From a662ecf4fc503d52f454e33f8bd7b226ec0a1c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:33:25 -0300 Subject: [PATCH 1/8] fix(validator): re-send proposer preparations every slot, not once an epoch A beacon node keeps prepare_beacon_proposer's fee recipients in memory, so a restart forgets them, and until the next call it builds its own payloads for the zero address. The validator client sent them only from the epoch refresh, so a beacon node restarted mid-epoch spent the rest of that epoch paying its local blocks' execution-layer rewards to nobody. The duty loop now re-sends them at the end of every slot (skipping the slot whose refresh just sent them), off the slot loop like the registration sends, as Lighthouse's validator client does. The settings are read on each send, so a keymanager fee-recipient change reaches the node within a slot. Found on a kurtosis devnet: an ethlambda beacon node SIGKILLed 14 slots into an epoch built 6 local blocks for the zero address before the next epoch's refresh (fulu), and 3 more after the same fault on gloas; the validator client logged "Block pays its execution-layer rewards to an address this client did not ask for" for each. --- crates/validator/src/lib.rs | 175 ++++++++++++++++++++++++++++++------ docs/rpc.md | 4 +- docs/spec_deviations.md | 4 +- 3 files changed, 155 insertions(+), 28 deletions(-) diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 4117af702..2a8ee775e 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -59,14 +59,14 @@ use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, use tokio::sync::{Mutex, RwLock}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use crate::aggregation::AggregationService; use crate::attestation::AttestationService; -use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::ProposerPreparationDto; use crate::beacon_node::fallback::FallbackBeaconNode; use crate::beacon_node::http::HttpBeaconNode; +use crate::beacon_node::{BeaconNodeApi, ValidatorEntry}; use crate::duties::DutiesService; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; @@ -349,6 +349,9 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // The slot whose duties were last attempted, so an overrunning slot does // not cost the next one as well. See `SlotClock::next_slot_to_serve`. let mut served: Option = None; + // The validators the last refresh resolved, whose proposer + // preparations are re-sent every slot (see `prepare_proposers`). + let mut resolved: Arc> = Arc::default(); loop { // One wake per slot, at the slot boundary, rather than one at the @@ -376,6 +379,8 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // resolved. let mut resubscribe = false; let mut to_register = None; + // Whether this slot's refresh already sent the preparations. + let mut prepared = false; if last_epoch != Some(epoch) { // Snapshot the pubkeys and let the guard drop right here, before // `refresh_epoch`'s awaits (a sync check, two duties fetches, a @@ -398,10 +403,16 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &context, ); match refreshed.await { - Ok(Refreshed { moved, registrable }) => { + Ok(Refreshed { + moved, + registrable, + resolved: refreshed_entries, + }) => { last_epoch = Some(epoch); resubscribe = moved; to_register = Some(registrable); + resolved = Arc::new(refreshed_entries); + prepared = true; } // A beacon node that is down, syncing or answering badly will // likely answer next epoch, so keep the schedule already held @@ -516,6 +527,26 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { if let Some(registration) = ®istration { tokio::spawn(registration.clone().register_changed(epoch, store.clone())); } + + // The proposer preparations again, for a beacon node that restarted + // since the refresh sent them and so builds for the zero address + // until it hears from this client. At the end of the slot, so they + // reach the node before the next slot's proposal rather than racing + // this one's, and off the slot loop like the sends above. Skipped + // on the slot whose refresh just sent them. A failure is only + // logged at debug: the node is down or restarting, every other + // duty of the slot is already warning about it, and the next slot + // tries again. + if !prepared && !resolved.is_empty() { + let (beacon_node, settings, resolved) = + (beacon_node.clone(), settings.clone(), resolved.clone()); + tokio::spawn(async move { + let sent = prepare_proposers(beacon_node.as_ref(), &resolved, &settings); + let _ = sent.await.inspect_err(|err| { + debug!(%slot, %err, "Failed to re-send this client's fee recipients"); + }); + }); + } } }; @@ -940,6 +971,10 @@ struct Refreshed { /// node resolved to an index whose validator is pending or active (see /// [`is_registrable`]). registrable: Vec, + /// Every validator the beacon node resolved to an index, which the duty + /// loop re-sends the proposer preparations of each slot until the next + /// refresh (see [`prepare_proposers`]). + resolved: Vec, } /// Resolve any newly activated validators and refresh the duty schedule. @@ -986,34 +1021,14 @@ async fn refresh_epoch( metrics::set_validators_resolved(indices.len() as u64); duties.set_indices(indices); - // Re-sent every epoch, not once at startup, because the node forgets. - // - // A preparation is kept for the epoch it arrived in and two more, and is - // lost entirely when the node restarts. A client that sent this once would - // stop being registered a few minutes later with nothing reporting it, and - // would find out only by proposing a block that paid someone else. + // Sent here, ahead of the epoch's duties, and then again every slot by the + // duty loop (see `prepare_proposers`). // // A failure is logged rather than propagated. The consequence is a payload // built for the wrong address, which the check before signing catches; the // consequence of returning here would be losing this epoch's attester // duties, which is worse and unrelated. - // - // Each validator is registered with its own address, and one with none, - // neither its own nor the default, is left out: there is nothing to tell - // the node, which then picks one itself, as the startup warning says. - let preparations: Vec = entries - .iter() - .filter_map(|entry| { - let fee_recipient = settings.fee_recipient(&entry.pubkey)?; - Some(ProposerPreparationDto { - validator_index: entry.index, - fee_recipient: crate::beacon_node::dto::encode_hex(&fee_recipient.0), - }) - }) - .collect(); - if !preparations.is_empty() - && let Err(err) = beacon_node.prepare_beacon_proposer(&preparations).await - { + if let Err(err) = prepare_proposers(beacon_node.as_ref(), &entries, settings).await { warn!( %epoch, %err, @@ -1031,9 +1046,47 @@ async fn refresh_epoch( Ok(Refreshed { moved: changed, registrable: registrable.map(|entry| entry.pubkey).collect(), + resolved: entries, }) } +/// Tell the beacon node the address each of `entries`' blocks pays: one +/// `prepare_beacon_proposer` call, or none when no validator has an address. +/// +/// Sent every slot, as Lighthouse's validator client does, not once an epoch. +/// The node keeps a preparation for the epoch it arrived in and two more, and +/// loses all of them when it restarts; until the next one arrives it builds +/// its own payloads for the zero address. Sent once an epoch, a restart +/// mid-epoch cost the rest of that epoch's local blocks their execution-layer +/// rewards: six blocks on a devnet, from a beacon node killed 14 slots into its +/// epoch. Sent every slot, it costs at most the next slot's. +/// +/// Each validator is registered with its own address, and one with none, +/// neither its own nor the default, is left out: there is nothing to tell the +/// node, which then picks one itself, as the startup warning says. Read from +/// `settings` on every call, so a keymanager change reaches the node within a +/// slot. +async fn prepare_proposers( + beacon_node: &B, + entries: &[ValidatorEntry], + settings: &ProposerSettings, +) -> Result<()> { + let preparations: Vec = entries + .iter() + .filter_map(|entry| { + let fee_recipient = settings.fee_recipient(&entry.pubkey)?; + Some(ProposerPreparationDto { + validator_index: entry.index, + fee_recipient: crate::beacon_node::dto::encode_hex(&fee_recipient.0), + }) + }) + .collect(); + if preparations.is_empty() { + return Ok(()); + } + beacon_node.prepare_beacon_proposer(&preparations).await +} + /// Whether a validator whose Beacon API `status` is this one is sent a builder /// registration: one that is pending or active, not one that exited or was /// withdrawn. @@ -1221,6 +1274,76 @@ mod tests { assert_eq!(node.preparations().len(), 4); } + /// The refresh hands the duty loop the validators it resolved, and the + /// loop's per-slot send registers exactly what the refresh did. That send + /// is what a beacon node restarted mid-epoch hears from first: once an + /// epoch, its local blocks paid the zero address for the rest of it. + #[tokio::test] + async fn the_per_slot_send_repeats_the_refreshs_preparations() { + let node = Arc::new(node()); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + let settings = settings(Some(H160([0xab; 20]))); + let keys = [pubkey(1), pubkey(2)]; + let Refreshed { resolved, .. } = + refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) + .await + .expect("refreshes"); + let indices: Vec = resolved.iter().map(|entry| entry.index).collect(); + assert_eq!(indices, vec![11, 22]); + let by_refresh = node.preparations(); + + prepare_proposers(node.as_ref(), &resolved, &settings) + .await + .expect("sends"); + let sent = node.preparations(); + assert_eq!(sent.len(), 4); + let pairs = |entries: &[ProposerPreparationDto]| -> Vec<(u64, String)> { + entries + .iter() + .map(|entry| (entry.validator_index, entry.fee_recipient.clone())) + .collect() + }; + assert_eq!(pairs(&sent[2..]), pairs(&by_refresh)); + } + + /// The per-slot send reads the settings each time, so a fee recipient the + /// keymanager changes mid-epoch reaches the node within a slot rather than + /// at the next refresh. + #[tokio::test] + async fn the_per_slot_send_carries_a_fee_recipient_changed_since_the_refresh() { + let node = Arc::new(node()); + let settings = settings(Some(H160([0xab; 20]))); + let resolved = vec![ValidatorEntry { + index: 11, + pubkey: pubkey(1), + status: "active_ongoing".to_string(), + }]; + settings.set_fee_recipient(&pubkey(1), H160([0xcd; 20])); + + prepare_proposers(node.as_ref(), &resolved, &settings) + .await + .expect("sends"); + let sent = node.preparations(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].fee_recipient, format!("0x{}", "cd".repeat(20))); + } + + /// No address at all is nothing to send, not an empty request. + #[tokio::test] + async fn the_per_slot_send_sends_nothing_without_an_address() { + let node = Arc::new(node()); + let resolved = vec![ValidatorEntry { + index: 11, + pubkey: pubkey(1), + status: "active_ongoing".to_string(), + }]; + + prepare_proposers(node.as_ref(), &resolved, &settings(None)) + .await + .expect("nothing to send is not a failure"); + assert!(node.preparations().is_empty()); + } + /// The refresh says when the attester schedule moved, which is what owes /// the node its subnet subscriptions, and sends none itself: they wait /// for the slot's block, since each costs a remote key a round trip to diff --git a/docs/rpc.md b/docs/rpc.md index e023d89fb..2e4521b91 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -410,7 +410,9 @@ the chain actor writes, so no request waits on the actor. next slot's block is asked for. Entries more than an epoch old are dropped once a slot. - **`prepare_beacon_proposer`** records each validator's fee recipient, in - memory (a validator client repeats the call every epoch). + memory, so a restart forgets it and the node builds its own payloads for the + zero address until the next call. ethlambda's validator client repeats the + call every slot, as Lighthouse's does. - **`register_validator`** takes a JSON array or an SSZ list of signed builder registrations. Registrations of validators the head state does not hold, or holds as exited, are dropped and counted diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index b36d39c75..7b747dc03 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -839,4 +839,6 @@ signature, whichever way the node answered. Three departures: node holds them in memory only and gossip does not replay them, so a node that restarts would otherwise self-build without them, and ignore builders' bids, until the next dependent - root. Same cadence as `prepare_beacon_proposer`, which a restart loses too. + root. `prepare_beacon_proposer`, which a restart loses too, is re-sent every + slot instead: it is one small request, where a preference is a signature per + proposal. From 76d43fce80cf4e7d34e1b2772fcac1a5910ec59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:49:28 -0300 Subject: [PATCH 2/8] fix(validator): add a quiet prepare_beacon_proposer for the per-slot send The per-slot re-send went through FallbackBeaconNode::prepare_beacon_proposer, the loud try_all: with a node down it warned for each refusing node every slot and, with none accepting, claimed a missed duty on slots with no duty. prepare_beacon_proposer_quietly is the same request through try_all_quiet, as submit_proposer_preferences does; its default implementation is the plain call, for a single node. --- crates/validator/src/beacon_node/fallback.rs | 49 ++++++++++++++++++++ crates/validator/src/beacon_node/mod.rs | 13 ++++++ 2 files changed, 62 insertions(+) diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 24a199095..90b83657b 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -647,6 +647,22 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Every node too, quietly: the duty loop sends this again each slot and + /// logs its own failure, so a node that is down costs a `debug!` line a + /// slot rather than two warnings, one of them claiming a missed duty. + async fn prepare_beacon_proposer_quietly( + &self, + preparations: &[ProposerPreparationDto], + ) -> Result<()> { + self.try_all_quiet( + "prepare_beacon_proposer", + "it keeps the fee recipients it was last given", + |node| node.prepare_beacon_proposer(preparations), + ) + .await + .map(|_| ()) + } + /// Every node, for the same reason, and with more at stake. A subscription /// is what puts a node on the attestation subnets and what makes it collect /// the votes an aggregate is folded from, so a node that never received one @@ -1108,6 +1124,39 @@ mod tests { assert_eq!(fallback.nodes[1].preparations().len(), 1); } + /// The per-slot send goes to every node like the epoch's, and fails only + /// when none accepted. What it changes is the logging, which this does + /// not see. + #[tokio::test] + async fn the_quiet_fee_recipient_send_reaches_every_node_too() { + let preparations = [ProposerPreparationDto { + validator_index: 1, + fee_recipient: "0xab".to_string(), + }]; + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::failing("down"), + MockBeaconNode::new(), + MockBeaconNode::new(), + ]); + fallback + .prepare_beacon_proposer_quietly(&preparations) + .await + .expect("two nodes accepted"); + assert!(fallback.nodes[0].preparations().is_empty()); + assert_eq!(fallback.nodes[1].preparations().len(), 1); + assert_eq!(fallback.nodes[2].preparations().len(), 1); + + let down = FallbackBeaconNode::new(vec![ + MockBeaconNode::failing("down"), + MockBeaconNode::failing("down"), + ]); + let err = down + .prepare_beacon_proposer_quietly(&preparations) + .await + .expect_err("no node accepted"); + assert!(matches!(err, Error::AllBeaconNodesFailed(_)), "got {err:?}"); + } + /// One unreachable node must not stop the reachable ones being told. #[tokio::test] async fn one_failing_node_does_not_stop_the_others_being_subscribed() { diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 49e3b5890..ec99c6f3f 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -734,6 +734,19 @@ pub trait BeaconNodeApi: Send + Sync { /// before it is signed rather than assumed. async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()>; + /// [`Self::prepare_beacon_proposer`] for a caller that sends it again + /// every slot and reports a failure itself. The same request; what + /// differs is the logging around it. The fallback logs a refusing node at + /// `debug!` and says nothing when none accepts, where the plain call warns + /// for each refusal and then that the duty was likely missed: once a slot, + /// for as long as a node is down, and on slots with no duty. + async fn prepare_beacon_proposer_quietly( + &self, + preparations: &[ProposerPreparationDto], + ) -> Result<()> { + self.prepare_beacon_proposer(preparations).await + } + /// Tells the node which committees this client's validators care about /// this epoch, so it can manage subnet subscriptions on their behalf. async fn subscribe_committees(&self, subscriptions: &[CommitteeSubscriptionDto]) -> Result<()>; From ec76cf7234a691c040b4cceb9896b35bc2e60cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:49:28 -0300 Subject: [PATCH 3/8] fix(validator): re-send fee recipients one at a time, quietly, for held keys only The per-slot re-send moves into Preparations, whose after_slot is the duty loop's gate: - it sends only the resolved validators the store still holds, so a key deleted through the keymanager is no longer prepared with --suggested-fee-recipient; - it is single-flight (InFlight), so a send that read an older fee recipient cannot land after a newer one; - it goes through prepare_beacon_proposer_quietly and logs its failure at debug. Refreshed carries whether the refresh's own send reached a node, so a failed one is re-sent on its own slot, and its warning no longer says the whole epoch may pay elsewhere. Tests drive the gate with the refresh's real outcome, and cover a deleted key and a re-send still waiting on a slow node. The two that repeated a_validators_own_address_is_registered_for_it and no_configured_address_sends_no_registration are gone. --- crates/validator/src/beacon_node/mock.rs | 20 ++ crates/validator/src/lib.rs | 432 +++++++++++++++++------ 2 files changed, 335 insertions(+), 117 deletions(-) diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 9bde4d283..034db6ea5 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -124,6 +124,12 @@ pub struct MockBeaconNode { /// records its batch, so a test releases calls one at a time instead of /// guessing how long a delay must be. pub registration_gate: Option>, + /// When set, each `prepare_beacon_proposer` call takes a permit before it + /// records its preparations and answers, as a slow node does, for as long + /// as the test holds it back. + pub preparation_gate: Option>, + /// Notified when `prepare_beacon_proposer` is called, before the gate. + pub preparation_entered: Arc, /// Every blinded block published, as (fork, body). pub published_blinded: Mutex)>>, /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; @@ -426,6 +432,13 @@ impl MockBeaconNode { self } + /// Make each `prepare_beacon_proposer` call wait for a permit from `gate` + /// before it records its preparations. + pub fn with_preparation_gate(mut self, gate: Arc) -> Self { + self.preparation_gate = Some(gate); + self + } + /// Make `produce_block` wait for a permit from `gate` before it answers. pub fn with_produce_gate(mut self, gate: Arc) -> Self { self.produce_gate = Some(gate); @@ -1215,6 +1228,13 @@ impl BeaconNodeApi for MockBeaconNode { async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()> { self.guard("prepare_beacon_proposer")?; + self.preparation_entered.notify_one(); + if let Some(gate) = &self.preparation_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } self.preparations .lock() .expect("lock") diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 9bc5b61e9..4d42e080c 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -68,6 +68,7 @@ use crate::beacon_node::fallback::FallbackBeaconNode; use crate::beacon_node::http::HttpBeaconNode; use crate::beacon_node::{BeaconNodeApi, ValidatorEntry}; use crate::duties::DutiesService; +use crate::in_flight::InFlight; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; use crate::preferences::PreferencesService; @@ -298,6 +299,9 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { .with_clock(clock.clone()) .with_builder_min_bid(config.builder_min_bid) .with_preferences(preferences.clone()); + // The proposer preparations' per-slot re-send. The epoch's own send is + // `refresh_epoch`'s. + let preparations = Arc::new(Preparations::new(beacon_node.clone(), settings.clone())); let registration = config.builder.map(|builder| { info!( boost_factor = builder.boost_factor, @@ -364,7 +368,7 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // not cost the next one as well. See `SlotClock::next_slot_to_serve`. let mut served: Option = None; // The validators the last refresh resolved, whose proposer - // preparations are re-sent every slot (see `prepare_proposers`). + // preparations are re-sent every slot (see `Preparations`). let mut resolved: Arc> = Arc::default(); loop { @@ -393,7 +397,8 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // resolved. let mut resubscribe = false; let mut to_register = None; - // Whether this slot's refresh already sent the preparations. + // Whether this slot's refresh got the preparations to a node, so + // the slot owes no re-send. let mut prepared = false; if last_epoch != Some(epoch) { // Snapshot the pubkeys and let the guard drop right here, before @@ -421,12 +426,13 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { moved, registrable, resolved: refreshed_entries, + prepared: sent, }) => { last_epoch = Some(epoch); resubscribe = moved; to_register = Some(registrable); resolved = Arc::new(refreshed_entries); - prepared = true; + prepared = sent; } // A beacon node that is down, syncing or answering badly will // likely answer next epoch, so keep the schedule already held @@ -544,23 +550,13 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // The proposer preparations again, for a beacon node that restarted // since the refresh sent them and so builds for the zero address - // until it hears from this client. At the end of the slot, so they - // reach the node before the next slot's proposal rather than racing - // this one's, and off the slot loop like the sends above. Skipped - // on the slot whose refresh just sent them. A failure is only - // logged at debug: the node is down or restarting, every other - // duty of the slot is already warning about it, and the next slot - // tries again. - if !prepared && !resolved.is_empty() { - let (beacon_node, settings, resolved) = - (beacon_node.clone(), settings.clone(), resolved.clone()); - tokio::spawn(async move { - let sent = prepare_proposers(beacon_node.as_ref(), &resolved, &settings); - let _ = sent.await.inspect_err(|err| { - debug!(%slot, %err, "Failed to re-send this client's fee recipients"); - }); - }); - } + // until it hears from this client. Once the slot's duties are done, + // so they reach the node ahead of the next slot's proposal rather + // than racing this one's: a third of the way in on a slot with no + // duties, after the aggregation or the payload vote on one with + // them. Off the slot loop like the sends above. Skipped on the slot + // whose refresh got them to a node. + preparations.after_slot(slot, prepared, &resolved, &store); } }; @@ -987,8 +983,12 @@ struct Refreshed { registrable: Vec, /// Every validator the beacon node resolved to an index, which the duty /// loop re-sends the proposer preparations of each slot until the next - /// refresh (see [`prepare_proposers`]). + /// refresh (see [`Preparations`]). resolved: Vec, + /// Whether the refresh's own proposer preparation reached a node, or had + /// nothing to send. When it did not, the duty loop re-sends on this slot + /// too rather than waiting for the next. + prepared: bool, } /// Resolve any newly activated validators and refresh the duty schedule. @@ -1036,20 +1036,24 @@ async fn refresh_epoch( duties.set_indices(indices); // Sent here, ahead of the epoch's duties, and then again every slot by the - // duty loop (see `prepare_proposers`). + // duty loop (see `Preparations`). // // A failure is logged rather than propagated. The consequence is a payload // built for the wrong address, which the check before signing catches; the // consequence of returning here would be losing this epoch's attester - // duties, which is worse and unrelated. - if let Err(err) = prepare_proposers(beacon_node.as_ref(), &entries, settings).await { - warn!( - %epoch, - %err, - "Failed to register this client's fee recipients; blocks proposed this epoch may \ - pay somewhere else" - ); - } + // duties, which is worse and unrelated. The duty loop sends again once + // this slot's duties are done. + let prepared = prepare_proposers(beacon_node.as_ref(), &entries, settings) + .await + .inspect_err(|err| { + warn!( + %epoch, + %err, + "Failed to register this client's fee recipients; a block proposed before \ + the re-send after this slot's duties may pay somewhere else" + ); + }) + .is_ok(); let changed = duties.refresh_around(epoch).await?; refresh_ptc(duties, epoch, context).await; @@ -1061,32 +1065,36 @@ async fn refresh_epoch( moved: changed, registrable: registrable.map(|entry| entry.pubkey).collect(), resolved: entries, + prepared, }) } /// Tell the beacon node the address each of `entries`' blocks pays: one /// `prepare_beacon_proposer` call, or none when no validator has an address. -/// -/// Sent every slot, as Lighthouse's validator client does, not once an epoch. -/// The node keeps a preparation for the epoch it arrived in and two more, and -/// loses all of them when it restarts; until the next one arrives it builds -/// its own payloads for the zero address. Sent once an epoch, a restart -/// mid-epoch cost the rest of that epoch's local blocks their execution-layer -/// rewards: six blocks on a devnet, from a beacon node killed 14 slots into its -/// epoch. Sent every slot, it costs at most the next slot's. -/// -/// Each validator is registered with its own address, and one with none, -/// neither its own nor the default, is left out: there is nothing to tell the -/// node, which then picks one itself, as the startup warning says. Read from -/// `settings` on every call, so a keymanager change reaches the node within a -/// slot. +/// The epoch refresh's send; the duty loop's per-slot one is [`Preparations`]. async fn prepare_proposers( beacon_node: &B, entries: &[ValidatorEntry], settings: &ProposerSettings, ) -> Result<()> { - let preparations: Vec = entries - .iter() + let preparations = preparations(entries, settings); + if preparations.is_empty() { + return Ok(()); + } + beacon_node.prepare_beacon_proposer(&preparations).await +} + +/// One preparation for each of `entries` that has an address, its own or the +/// default, as `settings` names it now. +/// +/// A validator with neither is left out: there is nothing to tell the node, +/// which then picks an address itself, as the startup warning says. +fn preparations<'a>( + entries: impl IntoIterator, + settings: &ProposerSettings, +) -> Vec { + entries + .into_iter() .filter_map(|entry| { let fee_recipient = settings.fee_recipient(&entry.pubkey)?; Some(ProposerPreparationDto { @@ -1094,11 +1102,132 @@ async fn prepare_proposers( fee_recipient: crate::beacon_node::dto::encode_hex(&fee_recipient.0), }) }) - .collect(); - if preparations.is_empty() { - return Ok(()); + .collect() +} + +/// The proposer preparations, sent again once every slot's duties are done. +/// +/// # Every slot, not once an epoch +/// +/// As Lighthouse's validator client does. A node keeps a preparation for the +/// epoch it arrived in and two more, and loses all of them when it restarts; +/// until the next one arrives it builds its own payloads for the zero address. +/// Sent only by the epoch refresh, a restart mid-epoch cost the rest of that +/// epoch's local blocks their execution-layer rewards: six blocks on a devnet, +/// from a beacon node killed 14 slots into its epoch. Sent every slot, a +/// restart costs at most a proposal that comes before the first re-send the +/// node is back up for, which is within a slot of its return. +/// +/// # What is sent +/// +/// The validators the last refresh resolved that the store still holds, each +/// with the address [`ProposerSettings`] gives it at the send, so a +/// keymanager change to a fee recipient reaches the node after the slot's +/// duties. Three cases that does not cover: +/// +/// - **A key deleted through the keymanager** (`DELETE /eth/v1/keystores` or +/// `/remotekeys`) is not sent again. Its overrides are forgotten with it, so +/// it would otherwise go out with `--suggested-fee-recipient`, overwriting +/// the preparation of a client the key moved to on the same node. The node +/// keeps the address it was last given until it expires it or restarts. +/// - **A `DELETE` of a validator's fee recipient that leaves it no address** +/// (no `--suggested-fee-recipient` either) sends nothing for it, so the node +/// keeps the address it was last given, the same way. +/// - **A key imported mid-epoch** is not sent until the next refresh resolves +/// its index. Nothing is lost: that refresh is also what fetches its +/// proposer duties (they are narrowed to the indices it resolves, see +/// [`DutiesService::set_indices`]), so this client proposes no block for it +/// before the refresh that prepares it. +/// +/// # One send at a time +/// +/// Each send reads the settings, then waits on every node. Against a slow +/// node a send can outlive the next slot's, and two in flight could land in +/// either order, reverting a fee recipient changed in between. So a send that +/// finds the last one still running skips, and the slot after it sends what +/// is current then (see [`crate::in_flight`]). +/// +/// The epoch refresh's send does not take part. It runs on the duty loop +/// ahead of the epoch's first proposal, where skipping would cost that +/// proposal its preparation and waiting would cost it time. A re-send still +/// in flight from the slot before can then land after it, and the next +/// slot's re-send puts the current address back. +/// +/// # Quiet on failure +/// +/// A node that is down, restarting or refusing costs a `debug!` line, here +/// and per node (see [`BeaconNodeApi::prepare_beacon_proposer_quietly`]): the +/// slot's duties already report a node that is down, and the next slot tries +/// again. The refresh's send still warns. +struct Preparations { + beacon_node: Arc, + settings: Arc, + /// The re-send, while it runs. + in_flight: InFlight, +} + +impl Preparations { + fn new(beacon_node: Arc, settings: Arc) -> Self { + Self { + beacon_node, + settings, + in_flight: InFlight::default(), + } + } + + /// The duty loop's gate, once the slot's duties are done: spawn the + /// re-send of `resolved`'s preparations, unless the slot's refresh got + /// them to a node (`prepared`) or there is nobody to prepare. The task, + /// for a test to await. + fn after_slot( + self: &Arc, + slot: u64, + prepared: bool, + resolved: &Arc>, + store: &Arc>, + ) -> Option> { + if prepared || resolved.is_empty() { + return None; + } + let resend = self.clone().resend(slot, resolved.clone(), store.clone()); + Some(tokio::spawn(resend)) + } + + async fn resend( + self: Arc, + slot: u64, + resolved: Arc>, + store: Arc>, + ) { + // Before the settings are read, so the sends reach the nodes in the + // order they read them. + let Some(_running) = self.in_flight.enter() else { + debug!( + %slot, + "The last fee recipient re-send is still running; skipping this slot's" + ); + return; + }; + // Built under the store's read guard, which is released before the + // send, so a keymanager write waits on no node. + let preparations = { + let store = store.read().await; + let held = resolved + .iter() + .filter(|entry| store.contains(&entry.pubkey)); + preparations(held, &self.settings) + }; + if preparations.is_empty() { + return; + } + let sent = self + .beacon_node + .prepare_beacon_proposer_quietly(&preparations) + .await; + let _ = sent.inspect_err(|err| { + debug!(%slot, %err, "Failed to re-send this client's fee recipients"); + }); } - beacon_node.prepare_beacon_proposer(&preparations).await } /// Whether a validator whose Beacon API `status` is this one is sent a builder @@ -1259,10 +1388,12 @@ mod tests { .expect("refreshes"); } - /// One preparation per resolved validator, every epoch. The node keeps a - /// preparation for three epochs and forgets all of them on restart, so a - /// client that sent this once at startup would quietly stop being - /// registered. + /// One preparation per resolved validator from every refresh, ahead of + /// the epoch's duties. The node keeps a preparation for three epochs and + /// forgets all of them on restart, so a client that sent this once at + /// startup would quietly stop being registered; the duty loop also sends + /// them again after every other slot's duties (see + /// `the_preparations_go_out_again_after_every_slot_but_the_refreshs`). #[tokio::test] async fn every_resolved_validator_is_registered_for_its_fee_recipient() { let node = Arc::new(node()); @@ -1288,74 +1419,133 @@ mod tests { assert_eq!(node.preparations().len(), 4); } - /// The refresh hands the duty loop the validators it resolved, and the - /// loop's per-slot send registers exactly what the refresh did. That send - /// is what a beacon node restarted mid-epoch hears from first: once an - /// epoch, its local blocks paid the zero address for the rest of it. + /// A store holding `keys`, behind a signer nothing calls: these tests ask + /// the store which keys it holds, never for a signature. + fn store_holding(keys: &[BlsPubkey]) -> Arc> { + let mut store = ValidatorStore::new(); + for key in keys { + store + .insert_remote(*key, "http://127.0.0.1:9000") + .expect("inserts"); + } + Arc::new(RwLock::new(store)) + } + + /// Every preparation `node` was sent, as (index, address), in order. + fn sent(node: &MockBeaconNode) -> Vec<(u64, String)> { + node.preparations() + .into_iter() + .map(|entry| (entry.validator_index, entry.fee_recipient)) + .collect() + } + + /// The address whose twenty bytes are all `byte`, as a preparation + /// carries it. + fn address(byte: u8) -> String { + crate::beacon_node::dto::encode_hex(&[byte; 20]) + } + + /// The duty loop's gate. Nothing on the slot whose refresh got the + /// preparations to a node; after every other slot's duties, the + /// refresh's validators again, with the addresses the settings name by + /// then. That re-send is what a beacon node restarted mid-epoch hears + /// from first: sent only by the refresh, its local blocks paid the zero + /// address for the rest of the epoch. #[tokio::test] - async fn the_per_slot_send_repeats_the_refreshs_preparations() { + async fn the_preparations_go_out_again_after_every_slot_but_the_refreshs() { let node = Arc::new(node()); let mut duties = DutiesService::new(node.clone(), Vec::new()); - let settings = settings(Some(H160([0xab; 20]))); + let settings = Arc::new(settings(Some(H160([0xab; 20])))); let keys = [pubkey(1), pubkey(2)]; - let Refreshed { resolved, .. } = - refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) - .await - .expect("refreshes"); - let indices: Vec = resolved.iter().map(|entry| entry.index).collect(); - assert_eq!(indices, vec![11, 22]); - let by_refresh = node.preparations(); + let store = store_holding(&keys); + let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); - prepare_proposers(node.as_ref(), &resolved, &settings) + let Refreshed { + resolved, prepared, .. + } = refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) .await - .expect("sends"); - let sent = node.preparations(); - assert_eq!(sent.len(), 4); - let pairs = |entries: &[ProposerPreparationDto]| -> Vec<(u64, String)> { - entries - .iter() - .map(|entry| (entry.validator_index, entry.fee_recipient.clone())) - .collect() - }; - assert_eq!(pairs(&sent[2..]), pairs(&by_refresh)); + .expect("refreshes"); + assert!(prepared, "the node took the refresh's send"); + let resolved = Arc::new(resolved); + assert!( + preparations + .after_slot(96, prepared, &resolved, &store) + .is_none(), + "the refresh's slot owes no re-send" + ); + assert_eq!(sent(&node), [(11, address(0xab)), (22, address(0xab))]); + + settings.set_fee_recipient(&pubkey(2), H160([0xcd; 20])); + preparations + .after_slot(97, false, &resolved, &store) + .expect("every other slot re-sends") + .await + .expect("the re-send ends"); + assert_eq!(sent(&node)[2..], [(11, address(0xab)), (22, address(0xcd))]); } - /// The per-slot send reads the settings each time, so a fee recipient the - /// keymanager changes mid-epoch reaches the node within a slot rather than - /// at the next refresh. + /// A key the keymanager deleted after the refresh is not prepared again. + /// Its overrides went with it, so it would otherwise go out every slot + /// with the default address, overwriting the preparation of a client the + /// key moved to on the same node. #[tokio::test] - async fn the_per_slot_send_carries_a_fee_recipient_changed_since_the_refresh() { + async fn a_key_deleted_since_the_refresh_is_not_prepared_again() { let node = Arc::new(node()); - let settings = settings(Some(H160([0xab; 20]))); - let resolved = vec![ValidatorEntry { - index: 11, - pubkey: pubkey(1), - status: "active_ongoing".to_string(), - }]; - settings.set_fee_recipient(&pubkey(1), H160([0xcd; 20])); - - prepare_proposers(node.as_ref(), &resolved, &settings) + let settings = Arc::new(settings(Some(H160([0xab; 20])))); + settings.set_fee_recipient(&pubkey(2), H160([0xcd; 20])); + let store = store_holding(&[pubkey(1), pubkey(2)]); + let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); + let resolved = Arc::new(node.validators.clone()); + + // What a keystore or remote key `DELETE` does to the two. + store.write().await.remove(&pubkey(2)); + settings.forget(&pubkey(2)); + preparations + .after_slot(97, false, &resolved, &store) + .expect("due") .await - .expect("sends"); - let sent = node.preparations(); - assert_eq!(sent.len(), 1); - assert_eq!(sent[0].fee_recipient, format!("0x{}", "cd".repeat(20))); + .expect("the re-send ends"); + + assert_eq!(sent(&node), [(11, address(0xab))]); } - /// No address at all is nothing to send, not an empty request. + /// One re-send at a time. A slot whose re-send finds the last one still + /// waiting on a node skips, so a send that read an older fee recipient + /// cannot land after one that read the newer; the slot after sends what + /// is current then. #[tokio::test] - async fn the_per_slot_send_sends_nothing_without_an_address() { - let node = Arc::new(node()); - let resolved = vec![ValidatorEntry { - index: 11, - pubkey: pubkey(1), - status: "active_ongoing".to_string(), - }]; + async fn a_re_send_still_waiting_on_a_node_makes_the_next_slot_skip() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let node = Arc::new(node().with_preparation_gate(gate.clone())); + let settings = Arc::new(settings(Some(H160([0xab; 20])))); + let store = store_holding(&[pubkey(1), pubkey(2)]); + let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); + let resolved = Arc::new(node.validators.clone()); + + let slow = preparations + .after_slot(97, false, &resolved, &store) + .expect("due"); + node.preparation_entered.notified().await; + settings.set_fee_recipient(&pubkey(1), H160([0xcd; 20])); + let skipped = preparations + .after_slot(98, false, &resolved, &store) + .expect("due"); + tokio::time::timeout(std::time::Duration::from_secs(5), skipped) + .await + .expect("a skipped re-send does not wait on the node") + .expect("the skipped re-send ends"); + + gate.add_permits(1); + slow.await.expect("the slow re-send ends"); + assert_eq!(sent(&node), [(11, address(0xab)), (22, address(0xab))]); - prepare_proposers(node.as_ref(), &resolved, &settings(None)) + gate.add_permits(1); + preparations + .after_slot(99, false, &resolved, &store) + .expect("due") .await - .expect("nothing to send is not a failure"); - assert!(node.preparations().is_empty()); + .expect("the re-send ends"); + assert_eq!(sent(&node)[2..], [(11, address(0xcd)), (22, address(0xab))]); } /// The refresh says when the attester schedule moved, which is what owes @@ -1435,23 +1625,31 @@ mod tests { /// which the check before signing catches; the consequence of propagating /// it would be attesting on last epoch's schedule, which is worse and /// unrelated. + /// + /// It is reported, though, so the duty loop sends again on the refresh's + /// own slot rather than leaving the node unprepared until the next. #[tokio::test] async fn a_failed_registration_does_not_fail_the_refresh() { let node = Arc::new(node().failing_call("prepare_beacon_proposer", "node is unhappy")); let mut duties = DutiesService::new(node.clone(), Vec::new()); let keys = [pubkey(1), pubkey(2)]; + let settings = Arc::new(settings(Some(H160([0xab; 20])))); - refresh_epoch( - &node, - &mut duties, - &keys, - 3, - &settings(Some(H160([0xab; 20]))), - &context(), - ) - .await - .expect("the refresh must survive a failed registration"); + let Refreshed { + resolved, prepared, .. + } = refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) + .await + .expect("the refresh must survive a failed registration"); assert_eq!(duties.indices(), &[11, 22]); + assert!(!prepared, "no node took the refresh's send"); + + let preparations = Arc::new(Preparations::new(node.clone(), settings)); + let resend = + preparations.after_slot(96, prepared, &Arc::new(resolved), &store_holding(&keys)); + resend + .expect("the refresh's own slot re-sends") + .await + .expect("the re-send ends"); } /// A client with no keys must not ask the beacon node anything. From 133c4cec29f9eaa030b6041e294f023d6b5f3e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:49:28 -0300 Subject: [PATCH 4/8] docs(validator): say fee recipients are re-sent after every slot's duties The keymanager mismatch error, docs/cli.md, and the comments in proposer_settings.rs, preferences.rs and the beacon node's FeeRecipients still described the once-an-epoch cadence. Also says what a DELETE leaves the node with, and when an imported key is first sent. spec_deviations.md gave a signature per proposal as the reason preferences are re-sent less often, but a kept one is re-sent as signed: the difference is one request for every validator against one per preference. --- crates/net/rpc/src/beacon/validator.rs | 5 +++-- crates/validator/src/preferences.rs | 7 +++++-- crates/validator/src/proposal.rs | 7 ++++--- crates/validator/src/proposer_settings.rs | 10 ++++++---- docs/cli.md | 10 +++++++++- docs/spec_deviations.md | 5 +++-- 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index ac3294986..7ace6c9a2 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -332,8 +332,9 @@ async fn post_committee_subscriptions( /// Each validator's execution-layer fee recipient, as its validator client /// last named it. Read by block production, which puts the proposer's into -/// the payload attributes. In memory only: a validator client repeats the call -/// every epoch, so a restarted node relearns the map within one. +/// the payload attributes. In memory only: a validator client repeats the +/// call, ethlambda's and Lighthouse's every slot and some others only every +/// epoch, so a restarted node relearns the map within a slot or an epoch. pub(crate) type FeeRecipients = Arc>>; /// One entry of `prepare_beacon_proposer`. diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs index 250da6a26..b703004e7 100644 --- a/crates/validator/src/preferences.rs +++ b/crates/validator/src/preferences.rs @@ -58,8 +58,11 @@ //! A node keeps the preferences it accepted in memory only, so one that //! restarts has lost them, and gossip does not give them back. Every kept //! preference whose slot is still ahead is therefore sent again, as signed, -//! once an epoch, as the duty loop re-sends `prepare_beacon_proposer`, which a -//! restart loses too. The round runs at the poll of an epoch's second-to-last +//! once an epoch. `prepare_beacon_proposer`, which a restart loses too, is +//! re-sent every slot instead (see the duty loop's `Preparations`): that is +//! one request for every validator, where the kept preferences are one +//! request each (see "One preference per request" below). The round runs at +//! the poll of an epoch's second-to-last //! slot, or at the next poll after it when that one was skipped behind a slow //! node, and re-sends each kept preference whose slot has not started once //! the poll's duties are in. Normally that is the epoch's last slot and every diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index e62e2b16c..02afb66f5 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -323,9 +323,10 @@ impl ProposalService { "Block pays its execution-layer rewards to an address this client did not ask for; \ signing it anyway, since refusing would also forfeit the consensus reward and cost \ the network a slot. Check this beacon node's proposer preparation. A fee recipient \ - changed through the keymanager API reaches the node at the next epoch boundary; \ - from gloas, a proposer preference published before the change (by this process \ - before a restart, say) keeps the old address until the next dependent root." + changed through the keymanager API reaches the node after the duties of the slot \ + it was changed in, and one deleted to no address at all never does; from gloas, \ + a proposer preference published before the change (by this process before a \ + restart, say) keeps the old address until the next dependent root." ); crate::metrics::inc_fee_recipient_mismatches(); } diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 31e30321b..35c6dee3e 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -24,10 +24,12 @@ //! //! Graffiti and gas limit are read when they are used: the next proposal //! carries the new value. A fee recipient reaches the beacon node through -//! `prepare_beacon_proposer`, which the duty loop sends once per epoch, so a -//! change is registered at the next epoch boundary. A block proposed before -//! then pays the address the node was last told, which the check before -//! signing reports as a mismatch. +//! `prepare_beacon_proposer`, which the duty loop sends again once every +//! slot's duties are done, so a change is registered within a slot. A block +//! proposed before then pays the address the node was last told, which the +//! check before signing reports as a mismatch. A `DELETE` that leaves a +//! validator no address at all sends nothing for it, so the node keeps the +//! old one until it expires it or restarts. //! //! From gloas a node pays the address in the proposer preferences it holds for //! the slot instead, and gas limit likewise, and a preference cannot be diff --git a/docs/cli.md b/docs/cli.md index b04b4d89e..7aaff92b1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -615,6 +615,14 @@ entry and leaves the local key, and its per-validator settings, as they are. Its answer is `error` too, naming `/eth/v1/keystores`, and never `deleted`: the key is still signing here until that route removes it. +A deleted key's fee recipient is no longer sent to the beacon node, from the +next slot on. The node keeps the address it was last given for that validator +until it expires the entry or restarts, so a client the key moved to on the +same node prepares it with its own address without this one overwriting it. +An imported key's fee recipient is first sent at the next epoch boundary, when +its index is resolved, which is also when its proposer duties are first +fetched. + It also serves `POST /eth/v1/validator/{pubkey}/voluntary_exit`, which signs a voluntary exit for a validator this client holds, in a local keystore or in a remote signer, and answers with the `SignedVoluntaryExit`, for the exit's @@ -630,7 +638,7 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no registration, so the builder keeps paying the address it was last given. From gloas, the proposer preferences carry it, from the next dependent root on | +| `feerecipient` | `--suggested-fee-recipient` | Once the duties of the slot it is set in are done, when the fee recipients are sent to the beacon node again, as they are after every slot's duties. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no fee recipient and no registration, so the beacon node and the builder keep paying the address they were last given. From gloas, the proposer preferences carry it, from the next dependent root on | | `graffiti` | `--graffiti` | The next proposal | | `gas_limit` | `--gas-limit` | Before gloas, the next builder registration with `--builder-proposals` (usually at the next slot, later while the epoch's full registration send is still running, and again every slot until a node accepts it or the epoch's full send carries it), and stored and served only without it. From gloas, the proposer preferences' target gas limit, from the next dependent root on, flag or not | diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 7b747dc03..948dda188 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -840,5 +840,6 @@ signature, whichever way the node answered. Three departures: and gossip does not replay them, so a node that restarts would otherwise self-build without them, and ignore builders' bids, until the next dependent root. `prepare_beacon_proposer`, which a restart loses too, is re-sent every - slot instead: it is one small request, where a preference is a signature per - proposal. + slot instead: it is one request for every validator, where the kept + preferences are one request each, since a node answers a call carrying + several with one status for all of them. From 60650ca7693a5090d1ece684432649b18c2e6701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:55:47 -0300 Subject: [PATCH 5/8] docs(validator): fix the remaining once-an-epoch fee recipient text The keymanager's log after a fee recipient is set still said it reaches the beacon node at the next epoch. The mismatch error no longer mentions a validator left with no address, which never reaches that error, and the FeeRecipients comment no longer makes a claim about other validator clients' cadence. --- crates/net/rpc/src/beacon/validator.rs | 4 ++-- crates/validator/src/http_api/proposer.rs | 2 +- crates/validator/src/proposal.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index 7ace6c9a2..8ebfc8240 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -333,8 +333,8 @@ async fn post_committee_subscriptions( /// Each validator's execution-layer fee recipient, as its validator client /// last named it. Read by block production, which puts the proposer's into /// the payload attributes. In memory only: a validator client repeats the -/// call, ethlambda's and Lighthouse's every slot and some others only every -/// epoch, so a restarted node relearns the map within a slot or an epoch. +/// call (ethlambda's and Lighthouse's every slot), so a restarted node +/// relearns the map from its next one. pub(crate) type FeeRecipients = Arc>>; /// One entry of `prepare_beacon_proposer`. diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index 4fdf33f77..25cfd56de 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -313,7 +313,7 @@ pub async fn set_fee_recipient( info!( pubkey = %encode_hex(&pubkey.0), fee_recipient = %encode_hex(&address.0), - "Fee recipient set; registered with the beacon node at the next epoch" + "Fee recipient set; sent to the beacon node once this slot's duties are done" ); Ok(StatusCode::ACCEPTED) } diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 02afb66f5..f2f0f5998 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -324,9 +324,9 @@ impl ProposalService { signing it anyway, since refusing would also forfeit the consensus reward and cost \ the network a slot. Check this beacon node's proposer preparation. A fee recipient \ changed through the keymanager API reaches the node after the duties of the slot \ - it was changed in, and one deleted to no address at all never does; from gloas, \ - a proposer preference published before the change (by this process before a \ - restart, say) keeps the old address until the next dependent root." + it was changed in; from gloas, a proposer preference published before the change \ + (by this process before a restart, say) keeps the old address until the next \ + dependent root." ); crate::metrics::inc_fee_recipient_mismatches(); } From e2b58d4c08e3d375878dde49f542df11388a7be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:10:54 -0300 Subject: [PATCH 6/8] fix(validator): re-send fee recipients after the refresh's slot too The refresh's own slot skipped the per-slot re-send when the refresh's send reached a node. Two things could then stand past the next slot's proposal: a re-send from the slot before that landed after the refresh's send, carrying the address a keymanager change had replaced, and a change made after the refresh's send. Re-sending after every slot's duties puts the current address back before the next proposal, for one extra request an epoch. Refreshed loses its prepared field. The Preparations doc now says the HTTP client's 8 s request timeout is what keeps a hung node from stopping the re-sends to the healthy ones. --- crates/validator/src/lib.rs | 170 +++++++++++++++++------------------- 1 file changed, 82 insertions(+), 88 deletions(-) diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 4d42e080c..52dc07625 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -397,9 +397,6 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // resolved. let mut resubscribe = false; let mut to_register = None; - // Whether this slot's refresh got the preparations to a node, so - // the slot owes no re-send. - let mut prepared = false; if last_epoch != Some(epoch) { // Snapshot the pubkeys and let the guard drop right here, before // `refresh_epoch`'s awaits (a sync check, two duties fetches, a @@ -426,13 +423,11 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { moved, registrable, resolved: refreshed_entries, - prepared: sent, }) => { last_epoch = Some(epoch); resubscribe = moved; to_register = Some(registrable); resolved = Arc::new(refreshed_entries); - prepared = sent; } // A beacon node that is down, syncing or answering badly will // likely answer next epoch, so keep the schedule already held @@ -554,9 +549,9 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // so they reach the node ahead of the next slot's proposal rather // than racing this one's: a third of the way in on a slot with no // duties, after the aggregation or the payload vote on one with - // them. Off the slot loop like the sends above. Skipped on the slot - // whose refresh got them to a node. - preparations.after_slot(slot, prepared, &resolved, &store); + // them. Off the slot loop like the sends above. The refresh's slot + // too, although its refresh sent them: see `Preparations`. + preparations.after_slot(slot, &resolved, &store); } }; @@ -985,10 +980,6 @@ struct Refreshed { /// loop re-sends the proposer preparations of each slot until the next /// refresh (see [`Preparations`]). resolved: Vec, - /// Whether the refresh's own proposer preparation reached a node, or had - /// nothing to send. When it did not, the duty loop re-sends on this slot - /// too rather than waiting for the next. - prepared: bool, } /// Resolve any newly activated validators and refresh the duty schedule. @@ -1043,17 +1034,15 @@ async fn refresh_epoch( // consequence of returning here would be losing this epoch's attester // duties, which is worse and unrelated. The duty loop sends again once // this slot's duties are done. - let prepared = prepare_proposers(beacon_node.as_ref(), &entries, settings) - .await - .inspect_err(|err| { - warn!( - %epoch, - %err, - "Failed to register this client's fee recipients; a block proposed before \ - the re-send after this slot's duties may pay somewhere else" - ); - }) - .is_ok(); + let sent = prepare_proposers(beacon_node.as_ref(), &entries, settings).await; + let _ = sent.inspect_err(|err| { + warn!( + %epoch, + %err, + "Failed to register this client's fee recipients; a block proposed before the \ + re-send after this slot's duties may pay somewhere else" + ); + }); let changed = duties.refresh_around(epoch).await?; refresh_ptc(duties, epoch, context).await; @@ -1065,7 +1054,6 @@ async fn refresh_epoch( moved: changed, registrable: registrable.map(|entry| entry.pubkey).collect(), resolved: entries, - prepared, }) } @@ -1122,8 +1110,9 @@ fn preparations<'a>( /// /// The validators the last refresh resolved that the store still holds, each /// with the address [`ProposerSettings`] gives it at the send, so a -/// keymanager change to a fee recipient reaches the node after the slot's -/// duties. Three cases that does not cover: +/// keymanager change to a fee recipient reaches the node after the duties of +/// the slot it is made in, or the next slot's while a slow node still holds +/// the last send (see below). Three cases that does not cover: /// /// - **A key deleted through the keymanager** (`DELETE /eth/v1/keystores` or /// `/remotekeys`) is not sent again. Its overrides are forgotten with it, so @@ -1145,13 +1134,19 @@ fn preparations<'a>( /// node a send can outlive the next slot's, and two in flight could land in /// either order, reverting a fee recipient changed in between. So a send that /// finds the last one still running skips, and the slot after it sends what -/// is current then (see [`crate::in_flight`]). +/// is current then (see [`crate::in_flight`]). A node that never answers is +/// cut off by the HTTP client's request timeout, 8 s, so on a 12 s slot a +/// hung node can cost the healthy ones every other re-send, never all of +/// them. /// /// The epoch refresh's send does not take part. It runs on the duty loop /// ahead of the epoch's first proposal, where skipping would cost that -/// proposal its preparation and waiting would cost it time. A re-send still -/// in flight from the slot before can then land after it, and the next -/// slot's re-send puts the current address back. +/// proposal its preparation, and waiting on a hung node would cost it up to +/// that timeout. A re-send still in flight from the slot before can then land +/// after it, which is why the refresh's slot re-sends too, once its duties are +/// done, although its refresh just sent the same: the late send can then cost +/// at most the epoch's first proposal. If it is still running at that point, +/// that re-send skips and the next slot's puts the current address back. /// /// # Quiet on failure /// @@ -1175,18 +1170,17 @@ impl Preparations { } } - /// The duty loop's gate, once the slot's duties are done: spawn the - /// re-send of `resolved`'s preparations, unless the slot's refresh got - /// them to a node (`prepared`) or there is nobody to prepare. The task, - /// for a test to await. + /// The duty loop's gate, once every slot's duties are done, the + /// refresh's slot included: spawn the re-send of `resolved`'s + /// preparations, unless there is nobody to prepare. The task, for a test + /// to await; the loop leaves it detached, as it does its other sends. fn after_slot( self: &Arc, slot: u64, - prepared: bool, resolved: &Arc>, store: &Arc>, ) -> Option> { - if prepared || resolved.is_empty() { + if resolved.is_empty() { return None; } let resend = self.clone().resend(slot, resolved.clone(), store.clone()); @@ -1392,8 +1386,8 @@ mod tests { /// the epoch's duties. The node keeps a preparation for three epochs and /// forgets all of them on restart, so a client that sent this once at /// startup would quietly stop being registered; the duty loop also sends - /// them again after every other slot's duties (see - /// `the_preparations_go_out_again_after_every_slot_but_the_refreshs`). + /// them again after every slot's duties (see + /// `the_preparations_go_out_again_after_every_slot_the_refreshs_included`). #[tokio::test] async fn every_resolved_validator_is_registered_for_its_fee_recipient() { let node = Arc::new(node()); @@ -1445,43 +1439,55 @@ mod tests { crate::beacon_node::dto::encode_hex(&[byte; 20]) } - /// The duty loop's gate. Nothing on the slot whose refresh got the - /// preparations to a node; after every other slot's duties, the - /// refresh's validators again, with the addresses the settings name by - /// then. That re-send is what a beacon node restarted mid-epoch hears - /// from first: sent only by the refresh, its local blocks paid the zero - /// address for the rest of the epoch. + /// The duty loop's gate: after every slot's duties, the refresh's + /// included, the refresh's validators again, with the addresses the + /// settings name by then. That re-send is what a beacon node restarted + /// mid-epoch hears from first: sent only by the refresh, its local blocks + /// paid the zero address for the rest of the epoch. + /// + /// On the refresh's own slot it puts the current address back when the + /// refresh's send was overtaken, here by the previous slot's re-send + /// landing late with the address a keymanager change replaced. Skipped + /// there, that address would stand until after the next slot's duties, + /// past the next proposal. #[tokio::test] - async fn the_preparations_go_out_again_after_every_slot_but_the_refreshs() { + async fn the_preparations_go_out_again_after_every_slot_the_refreshs_included() { let node = Arc::new(node()); let mut duties = DutiesService::new(node.clone(), Vec::new()); let settings = Arc::new(settings(Some(H160([0xab; 20])))); + settings.set_fee_recipient(&pubkey(2), H160([0xcd; 20])); let keys = [pubkey(1), pubkey(2)]; let store = store_holding(&keys); let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); - let Refreshed { - resolved, prepared, .. - } = refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) - .await - .expect("refreshes"); - assert!(prepared, "the node took the refresh's send"); + let Refreshed { resolved, .. } = + refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) + .await + .expect("refreshes"); + assert_eq!(sent(&node), [(11, address(0xab)), (22, address(0xcd))]); + // The previous slot's re-send, which read validator 22's address + // before the change, reaching the node after the refresh's send. + let late = ProposerPreparationDto { + validator_index: 22, + fee_recipient: address(0xab), + }; + node.preparations.lock().expect("lock").push(late); + let resolved = Arc::new(resolved); - assert!( - preparations - .after_slot(96, prepared, &resolved, &store) - .is_none(), - "the refresh's slot owes no re-send" - ); - assert_eq!(sent(&node), [(11, address(0xab)), (22, address(0xab))]); + preparations + .after_slot(96, &resolved, &store) + .expect("the refresh's slot re-sends too") + .await + .expect("the re-send ends"); + assert_eq!(sent(&node)[3..], [(11, address(0xab)), (22, address(0xcd))]); - settings.set_fee_recipient(&pubkey(2), H160([0xcd; 20])); + settings.set_fee_recipient(&pubkey(2), H160([0xef; 20])); preparations - .after_slot(97, false, &resolved, &store) - .expect("every other slot re-sends") + .after_slot(97, &resolved, &store) + .expect("and so does every slot after it") .await .expect("the re-send ends"); - assert_eq!(sent(&node)[2..], [(11, address(0xab)), (22, address(0xcd))]); + assert_eq!(sent(&node)[5..], [(11, address(0xab)), (22, address(0xef))]); } /// A key the keymanager deleted after the refresh is not prepared again. @@ -1501,7 +1507,7 @@ mod tests { store.write().await.remove(&pubkey(2)); settings.forget(&pubkey(2)); preparations - .after_slot(97, false, &resolved, &store) + .after_slot(97, &resolved, &store) .expect("due") .await .expect("the re-send ends"); @@ -1522,14 +1528,10 @@ mod tests { let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); let resolved = Arc::new(node.validators.clone()); - let slow = preparations - .after_slot(97, false, &resolved, &store) - .expect("due"); + let slow = preparations.after_slot(97, &resolved, &store).expect("due"); node.preparation_entered.notified().await; settings.set_fee_recipient(&pubkey(1), H160([0xcd; 20])); - let skipped = preparations - .after_slot(98, false, &resolved, &store) - .expect("due"); + let skipped = preparations.after_slot(98, &resolved, &store).expect("due"); tokio::time::timeout(std::time::Duration::from_secs(5), skipped) .await .expect("a skipped re-send does not wait on the node") @@ -1541,7 +1543,7 @@ mod tests { gate.add_permits(1); preparations - .after_slot(99, false, &resolved, &store) + .after_slot(99, &resolved, &store) .expect("due") .await .expect("the re-send ends"); @@ -1625,31 +1627,23 @@ mod tests { /// which the check before signing catches; the consequence of propagating /// it would be attesting on last epoch's schedule, which is worse and /// unrelated. - /// - /// It is reported, though, so the duty loop sends again on the refresh's - /// own slot rather than leaving the node unprepared until the next. #[tokio::test] async fn a_failed_registration_does_not_fail_the_refresh() { let node = Arc::new(node().failing_call("prepare_beacon_proposer", "node is unhappy")); let mut duties = DutiesService::new(node.clone(), Vec::new()); let keys = [pubkey(1), pubkey(2)]; - let settings = Arc::new(settings(Some(H160([0xab; 20])))); - let Refreshed { - resolved, prepared, .. - } = refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) - .await - .expect("the refresh must survive a failed registration"); + refresh_epoch( + &node, + &mut duties, + &keys, + 3, + &settings(Some(H160([0xab; 20]))), + &context(), + ) + .await + .expect("the refresh must survive a failed registration"); assert_eq!(duties.indices(), &[11, 22]); - assert!(!prepared, "no node took the refresh's send"); - - let preparations = Arc::new(Preparations::new(node.clone(), settings)); - let resend = - preparations.after_slot(96, prepared, &Arc::new(resolved), &store_holding(&keys)); - resend - .expect("the refresh's own slot re-sends") - .await - .expect("the re-send ends"); } /// A client with no keys must not ask the beacon node anything. From 2faeed944882f500195cede87168bbfa3a9ab7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:10:54 -0300 Subject: [PATCH 7/8] docs(validator): qualify the fee recipient re-send timing and its request load A change reaches the node a slot later while a slow node holds the last send. docs/cli.md says the addresses go out as one request per slot to each beacon node. The default prepare_beacon_proposer_quietly says why it is the plain call, the mock's preparation_entered notes the permit every call stores, and a paragraph in preferences.rs is re-wrapped. --- crates/validator/src/beacon_node/mock.rs | 3 +++ crates/validator/src/beacon_node/mod.rs | 4 ++++ crates/validator/src/http_api/proposer.rs | 3 ++- crates/validator/src/preferences.rs | 11 ++++++----- crates/validator/src/proposer_settings.rs | 4 +++- docs/cli.md | 4 ++-- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 034db6ea5..79b9a70d6 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -129,6 +129,9 @@ pub struct MockBeaconNode { /// as the test holds it back. pub preparation_gate: Option>, /// Notified when `prepare_beacon_proposer` is called, before the gate. + /// Every call stores a permit (`notify_one`), so a test that awaits it + /// must not make an earlier call it does not mean to wait for: that call's + /// permit would answer the wait. pub preparation_entered: Arc, /// Every blinded block published, as (fork, body). pub published_blinded: Mutex)>>, diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index ec99c6f3f..bdfe5d35b 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -740,6 +740,10 @@ pub trait BeaconNodeApi: Send + Sync { /// `debug!` and says nothing when none accepts, where the plain call warns /// for each refusal and then that the duty was likely missed: once a slot, /// for as long as a node is down, and on slots with no duty. + /// + /// The default is the plain call because the logging is the fallback's + /// alone: one node (`HttpBeaconNode`) logs nothing about its answer either + /// way, so it has nothing to quieten. async fn prepare_beacon_proposer_quietly( &self, preparations: &[ProposerPreparationDto], diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index 25cfd56de..57a1fc368 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -313,7 +313,8 @@ pub async fn set_fee_recipient( info!( pubkey = %encode_hex(&pubkey.0), fee_recipient = %encode_hex(&address.0), - "Fee recipient set; sent to the beacon node once this slot's duties are done" + "Fee recipient set; sent to the beacon node once this slot's duties are done, or the \ + next slot's while a slow node holds the last send" ); Ok(StatusCode::ACCEPTED) } diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs index b703004e7..0fa9d5574 100644 --- a/crates/validator/src/preferences.rs +++ b/crates/validator/src/preferences.rs @@ -61,11 +61,12 @@ //! once an epoch. `prepare_beacon_proposer`, which a restart loses too, is //! re-sent every slot instead (see the duty loop's `Preparations`): that is //! one request for every validator, where the kept preferences are one -//! request each (see "One preference per request" below). The round runs at -//! the poll of an epoch's second-to-last -//! slot, or at the next poll after it when that one was skipped behind a slow -//! node, and re-sends each kept preference whose slot has not started once -//! the poll's duties are in. Normally that is the epoch's last slot and every +//! request each (see "One preference per request" below). +//! +//! The round runs at the poll of an epoch's second-to-last slot, or at the +//! next poll after it when that one was skipped behind a slow node, and +//! re-sends each kept preference whose slot has not started once the poll's +//! duties are in. Normally that is the epoch's last slot and every //! slot of the next one, so a node restarted mid-epoch lacks the preferences //! of the slots before the round and holds every later one. One restarted //! after the round lacks the next epoch's too, all but its last slot's, which diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 35c6dee3e..4e56f2f9e 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -25,7 +25,9 @@ //! Graffiti and gas limit are read when they are used: the next proposal //! carries the new value. A fee recipient reaches the beacon node through //! `prepare_beacon_proposer`, which the duty loop sends again once every -//! slot's duties are done, so a change is registered within a slot. A block +//! slot's duties are done, so a change is registered after the duties of the +//! slot it is made in (the next slot's while a slow node holds the last +//! send). A block //! proposed before then pays the address the node was last told, which the //! check before signing reports as a mismatch. A `DELETE` that leaves a //! validator no address at all sends nothing for it, so the node keeps the diff --git a/docs/cli.md b/docs/cli.md index 7aaff92b1..6962d20eb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -305,7 +305,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--http-address` | `127.0.0.1` | Bind address for the metrics and keymanager servers | | `--metrics-port` | `5064` | Prometheus metrics port | | `--keymanager-port` | `5062` | Keymanager API port. Only bound with `--enable-keymanager` | -| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours. The zero address is accepted, as every other client accepts it, but reported at error at startup: it burns the rewards, and builder registrations and proposer preferences sign it for the builder to pay | +| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours. The addresses reach each beacon node as one `prepare_beacon_proposer` request every slot, listing each validator that has an index on chain and an address, as Lighthouse's validator client sends it. The zero address is accepted, as every other client accepts it, but reported at error at startup: it burns the rewards, and builder registrations and proposer preferences sign it for the builder to pay | | `--gas-limit` | `60000000` | The gas limit every validator targets, unless the keymanager API gives it its own. Before gloas it goes into builder registrations (with `--builder-proposals`). From gloas it goes into the proposer preferences, which are published for every proposal, and the beacon node builds this client's own payloads toward it as well, so a network whose gas limit is not 60M needs it set, or every proposal pulls the limit toward 60M. **Sepolia and PlatÃ¥berget schedule `200000000`** (their `config.yaml`'s `GAS_LIMIT_SCHEDULE`, which this client does not read yet, [#795](https://github.com/lambdaclass/ethlambda/issues/795)), so on either pass `--gas-limit 200000000`. Must be positive | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, remote keys, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | @@ -638,7 +638,7 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | Once the duties of the slot it is set in are done, when the fee recipients are sent to the beacon node again, as they are after every slot's duties. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no fee recipient and no registration, so the beacon node and the builder keep paying the address they were last given. From gloas, the proposer preferences carry it, from the next dependent root on | +| `feerecipient` | `--suggested-fee-recipient` | Once the duties of the slot it is set in are done, when the fee recipients are sent to the beacon node again, as they are after every slot's duties (the next slot's while a slow beacon node still holds the last send). A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no fee recipient and no registration, so the beacon node and the builder keep paying the address they were last given. From gloas, the proposer preferences carry it, from the next dependent root on | | `graffiti` | `--graffiti` | The next proposal | | `gas_limit` | `--gas-limit` | Before gloas, the next builder registration with `--builder-proposals` (usually at the next slot, later while the epoch's full registration send is still running, and again every slot until a node accepts it or the epoch's full send carries it), and stored and served only without it. From gloas, the proposer preferences' target gas limit, from the next dependent root on, flag or not | From db6609434c581a54af7a1a19ac64ef0f1714d351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:31:26 -0300 Subject: [PATCH 8/8] docs(validator): say when an imported key's fee recipient is first sent The keymanager's log after a fee recipient is set, the feerecipient row in docs/cli.md and the proposer_settings.rs module doc said the address goes out after the slot's duties. That holds only for a validator the last epoch refresh resolved; one imported since, or with no index yet, is first sent once a refresh resolves it. The Preparations doc no longer bounds a hung node's cost only for 12 s slots, says a key imported again after a delete goes out with the defaults, and words after_slot's return plainly. The single-flight test asserts the slow re-send is still parked before the gate opens. --- crates/validator/src/http_api/proposer.rs | 4 ++-- crates/validator/src/lib.rs | 16 ++++++++++++---- crates/validator/src/proposer_settings.rs | 7 ++++--- docs/cli.md | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index 57a1fc368..2e292f818 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -313,8 +313,8 @@ pub async fn set_fee_recipient( info!( pubkey = %encode_hex(&pubkey.0), fee_recipient = %encode_hex(&address.0), - "Fee recipient set; sent to the beacon node once this slot's duties are done, or the \ - next slot's while a slow node holds the last send" + "Fee recipient set; sent to the beacon node with the next proposer preparation that \ + includes this validator" ); Ok(StatusCode::ACCEPTED) } diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 52dc07625..d368862d3 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -1119,6 +1119,8 @@ fn preparations<'a>( /// it would otherwise go out with `--suggested-fee-recipient`, overwriting /// the preparation of a client the key moved to on the same node. The node /// keeps the address it was last given until it expires it or restarts. +/// Imported again before the next refresh, the key is sent again, with the +/// defaults a key starts on once its overrides are forgotten. /// - **A `DELETE` of a validator's fee recipient that leaves it no address** /// (no `--suggested-fee-recipient` either) sends nothing for it, so the node /// keeps the address it was last given, the same way. @@ -1135,8 +1137,9 @@ fn preparations<'a>( /// either order, reverting a fee recipient changed in between. So a send that /// finds the last one still running skips, and the slot after it sends what /// is current then (see [`crate::in_flight`]). A node that never answers is -/// cut off by the HTTP client's request timeout, 8 s, so on a 12 s slot a -/// hung node can cost the healthy ones every other re-send, never all of +/// cut off by the HTTP client's request timeout, 8 s, so a hung node costs +/// the healthy ones only the re-sends that come due while a send waits on it: +/// at most every other one on 12 s slots, more on shorter ones, never all of /// them. /// /// The epoch refresh's send does not take part. It runs on the duty loop @@ -1172,8 +1175,9 @@ impl Preparations { /// The duty loop's gate, once every slot's duties are done, the /// refresh's slot included: spawn the re-send of `resolved`'s - /// preparations, unless there is nobody to prepare. The task, for a test - /// to await; the loop leaves it detached, as it does its other sends. + /// preparations, unless there is nobody to prepare. Returns the spawned + /// task's handle so a test can await it; the duty loop drops it, as it + /// does its other sends' handles. fn after_slot( self: &Arc, slot: u64, @@ -1536,6 +1540,10 @@ mod tests { .await .expect("a skipped re-send does not wait on the node") .expect("the skipped re-send ends"); + assert!( + node.preparations().is_empty(), + "the slow re-send is still parked at the gate, and the skipped one sent nothing" + ); gate.add_permits(1); slow.await.expect("the slow re-send ends"); diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 4e56f2f9e..1abb2853c 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -27,9 +27,10 @@ //! `prepare_beacon_proposer`, which the duty loop sends again once every //! slot's duties are done, so a change is registered after the duties of the //! slot it is made in (the next slot's while a slow node holds the last -//! send). A block -//! proposed before then pays the address the node was last told, which the -//! check before signing reports as a mismatch. A `DELETE` that leaves a +//! send). That is for a validator the last epoch refresh resolved; one +//! imported since is first sent once the next refresh resolves its index. A +//! block proposed before then pays the address the node was last told, which +//! the check before signing reports as a mismatch. A `DELETE` that leaves a //! validator no address at all sends nothing for it, so the node keeps the //! old one until it expires it or restarts. //! diff --git a/docs/cli.md b/docs/cli.md index 6962d20eb..91ee95a10 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -638,7 +638,7 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | Once the duties of the slot it is set in are done, when the fee recipients are sent to the beacon node again, as they are after every slot's duties (the next slot's while a slow beacon node still holds the last send). A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no fee recipient and no registration, so the beacon node and the builder keep paying the address they were last given. From gloas, the proposer preferences carry it, from the next dependent root on | +| `feerecipient` | `--suggested-fee-recipient` | Once the duties of the slot it is set in are done, when the fee recipients are sent to the beacon node again, as they are after every slot's duties (the next slot's while a slow beacon node still holds the last send). A validator imported since the last epoch boundary, or with no index on chain yet, is first sent once an epoch boundary resolves its index. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no fee recipient and no registration, so the beacon node and the builder keep paying the address they were last given. From gloas, the proposer preferences carry it, from the next dependent root on | | `graffiti` | `--graffiti` | The next proposal | | `gas_limit` | `--gas-limit` | Before gloas, the next builder registration with `--builder-proposals` (usually at the next slot, later while the epoch's full registration send is still running, and again every slot until a node accepts it or the epoch's full send carries it), and stored and served only without it. From gloas, the proposer preferences' target gas limit, from the next dependent root on, flag or not |