diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index ac3294986..8ebfc8240 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -332,8 +332,9 @@ async fn post_committee_subscriptions( /// Each validator's execution-layer fee recipient, as its validator client /// last named it. Read by block production, which puts the proposer's into -/// the payload attributes. In memory only: a validator client repeats the call -/// every epoch, so a restarted node relearns the map within one. +/// the payload attributes. In memory only: a validator client repeats the +/// call (ethlambda's and Lighthouse's every slot), so a restarted node +/// relearns the map from its next one. pub(crate) type FeeRecipients = Arc>>; /// One entry of `prepare_beacon_proposer`. diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 24a199095..90b83657b 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -647,6 +647,22 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Every node too, quietly: the duty loop sends this again each slot and + /// logs its own failure, so a node that is down costs a `debug!` line a + /// slot rather than two warnings, one of them claiming a missed duty. + async fn prepare_beacon_proposer_quietly( + &self, + preparations: &[ProposerPreparationDto], + ) -> Result<()> { + self.try_all_quiet( + "prepare_beacon_proposer", + "it keeps the fee recipients it was last given", + |node| node.prepare_beacon_proposer(preparations), + ) + .await + .map(|_| ()) + } + /// Every node, for the same reason, and with more at stake. A subscription /// is what puts a node on the attestation subnets and what makes it collect /// the votes an aggregate is folded from, so a node that never received one @@ -1108,6 +1124,39 @@ mod tests { assert_eq!(fallback.nodes[1].preparations().len(), 1); } + /// The per-slot send goes to every node like the epoch's, and fails only + /// when none accepted. What it changes is the logging, which this does + /// not see. + #[tokio::test] + async fn the_quiet_fee_recipient_send_reaches_every_node_too() { + let preparations = [ProposerPreparationDto { + validator_index: 1, + fee_recipient: "0xab".to_string(), + }]; + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::failing("down"), + MockBeaconNode::new(), + MockBeaconNode::new(), + ]); + fallback + .prepare_beacon_proposer_quietly(&preparations) + .await + .expect("two nodes accepted"); + assert!(fallback.nodes[0].preparations().is_empty()); + assert_eq!(fallback.nodes[1].preparations().len(), 1); + assert_eq!(fallback.nodes[2].preparations().len(), 1); + + let down = FallbackBeaconNode::new(vec![ + MockBeaconNode::failing("down"), + MockBeaconNode::failing("down"), + ]); + let err = down + .prepare_beacon_proposer_quietly(&preparations) + .await + .expect_err("no node accepted"); + assert!(matches!(err, Error::AllBeaconNodesFailed(_)), "got {err:?}"); + } + /// One unreachable node must not stop the reachable ones being told. #[tokio::test] async fn one_failing_node_does_not_stop_the_others_being_subscribed() { diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 9bde4d283..79b9a70d6 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -124,6 +124,15 @@ pub struct MockBeaconNode { /// records its batch, so a test releases calls one at a time instead of /// guessing how long a delay must be. pub registration_gate: Option>, + /// When set, each `prepare_beacon_proposer` call takes a permit before it + /// records its preparations and answers, as a slow node does, for as long + /// as the test holds it back. + pub preparation_gate: Option>, + /// Notified when `prepare_beacon_proposer` is called, before the gate. + /// Every call stores a permit (`notify_one`), so a test that awaits it + /// must not make an earlier call it does not mean to wait for: that call's + /// permit would answer the wait. + pub preparation_entered: Arc, /// Every blinded block published, as (fork, body). pub published_blinded: Mutex)>>, /// What `publish_blinded_block` reports. `None` is `Ok(HandedToBuilder)`; @@ -426,6 +435,13 @@ impl MockBeaconNode { self } + /// Make each `prepare_beacon_proposer` call wait for a permit from `gate` + /// before it records its preparations. + pub fn with_preparation_gate(mut self, gate: Arc) -> Self { + self.preparation_gate = Some(gate); + self + } + /// Make `produce_block` wait for a permit from `gate` before it answers. pub fn with_produce_gate(mut self, gate: Arc) -> Self { self.produce_gate = Some(gate); @@ -1215,6 +1231,13 @@ impl BeaconNodeApi for MockBeaconNode { async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()> { self.guard("prepare_beacon_proposer")?; + self.preparation_entered.notify_one(); + if let Some(gate) = &self.preparation_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } self.preparations .lock() .expect("lock") diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 49e3b5890..bdfe5d35b 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -734,6 +734,23 @@ pub trait BeaconNodeApi: Send + Sync { /// before it is signed rather than assumed. async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()>; + /// [`Self::prepare_beacon_proposer`] for a caller that sends it again + /// every slot and reports a failure itself. The same request; what + /// differs is the logging around it. The fallback logs a refusing node at + /// `debug!` and says nothing when none accepts, where the plain call warns + /// for each refusal and then that the duty was likely missed: once a slot, + /// for as long as a node is down, and on slots with no duty. + /// + /// The default is the plain call because the logging is the fallback's + /// alone: one node (`HttpBeaconNode`) logs nothing about its answer either + /// way, so it has nothing to quieten. + async fn prepare_beacon_proposer_quietly( + &self, + preparations: &[ProposerPreparationDto], + ) -> Result<()> { + self.prepare_beacon_proposer(preparations).await + } + /// Tells the node which committees this client's validators care about /// this epoch, so it can manage subnet subscriptions on their behalf. async fn subscribe_committees(&self, subscriptions: &[CommitteeSubscriptionDto]) -> Result<()>; diff --git a/crates/validator/src/http_api/proposer.rs b/crates/validator/src/http_api/proposer.rs index 4fdf33f77..2e292f818 100644 --- a/crates/validator/src/http_api/proposer.rs +++ b/crates/validator/src/http_api/proposer.rs @@ -313,7 +313,8 @@ pub async fn set_fee_recipient( info!( pubkey = %encode_hex(&pubkey.0), fee_recipient = %encode_hex(&address.0), - "Fee recipient set; registered with the beacon node at the next epoch" + "Fee recipient set; sent to the beacon node with the next proposer preparation that \ + includes this validator" ); Ok(StatusCode::ACCEPTED) } diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 6885c258d..d368862d3 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -59,15 +59,16 @@ use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, use tokio::sync::{Mutex, RwLock}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use crate::aggregation::AggregationService; use crate::attestation::AttestationService; -use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::ProposerPreparationDto; use crate::beacon_node::fallback::FallbackBeaconNode; use crate::beacon_node::http::HttpBeaconNode; +use crate::beacon_node::{BeaconNodeApi, ValidatorEntry}; use crate::duties::DutiesService; +use crate::in_flight::InFlight; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; use crate::preferences::PreferencesService; @@ -298,6 +299,9 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { .with_clock(clock.clone()) .with_builder_min_bid(config.builder_min_bid) .with_preferences(preferences.clone()); + // The proposer preparations' per-slot re-send. The epoch's own send is + // `refresh_epoch`'s. + let preparations = Arc::new(Preparations::new(beacon_node.clone(), settings.clone())); let registration = config.builder.map(|builder| { info!( boost_factor = builder.boost_factor, @@ -363,6 +367,9 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // The slot whose duties were last attempted, so an overrunning slot does // not cost the next one as well. See `SlotClock::next_slot_to_serve`. let mut served: Option = None; + // The validators the last refresh resolved, whose proposer + // preparations are re-sent every slot (see `Preparations`). + let mut resolved: Arc> = Arc::default(); loop { // One wake per slot, at the slot boundary, rather than one at the @@ -412,10 +419,15 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { &context, ); match refreshed.await { - Ok(Refreshed { moved, registrable }) => { + Ok(Refreshed { + moved, + registrable, + resolved: refreshed_entries, + }) => { last_epoch = Some(epoch); resubscribe = moved; to_register = Some(registrable); + resolved = Arc::new(refreshed_entries); } // A beacon node that is down, syncing or answering badly will // likely answer next epoch, so keep the schedule already held @@ -530,6 +542,16 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { if let Some(registration) = ®istration { tokio::spawn(registration.clone().register_changed(epoch, store.clone())); } + + // The proposer preparations again, for a beacon node that restarted + // since the refresh sent them and so builds for the zero address + // until it hears from this client. Once the slot's duties are done, + // so they reach the node ahead of the next slot's proposal rather + // than racing this one's: a third of the way in on a slot with no + // duties, after the aggregation or the payload vote on one with + // them. Off the slot loop like the sends above. The refresh's slot + // too, although its refresh sent them: see `Preparations`. + preparations.after_slot(slot, &resolved, &store); } }; @@ -954,6 +976,10 @@ struct Refreshed { /// node resolved to an index whose validator is pending or active (see /// [`is_registrable`]). registrable: Vec, + /// Every validator the beacon node resolved to an index, which the duty + /// loop re-sends the proposer preparations of each slot until the next + /// refresh (see [`Preparations`]). + resolved: Vec, } /// Resolve any newly activated validators and refresh the duty schedule. @@ -1000,41 +1026,23 @@ async fn refresh_epoch( metrics::set_validators_resolved(indices.len() as u64); duties.set_indices(indices); - // Re-sent every epoch, not once at startup, because the node forgets. - // - // A preparation is kept for the epoch it arrived in and two more, and is - // lost entirely when the node restarts. A client that sent this once would - // stop being registered a few minutes later with nothing reporting it, and - // would find out only by proposing a block that paid someone else. + // Sent here, ahead of the epoch's duties, and then again every slot by the + // duty loop (see `Preparations`). // // A failure is logged rather than propagated. The consequence is a payload // built for the wrong address, which the check before signing catches; the // consequence of returning here would be losing this epoch's attester - // duties, which is worse and unrelated. - // - // Each validator is registered with its own address, and one with none, - // neither its own nor the default, is left out: there is nothing to tell - // the node, which then picks one itself, as the startup warning says. - let preparations: Vec = entries - .iter() - .filter_map(|entry| { - let fee_recipient = settings.fee_recipient(&entry.pubkey)?; - Some(ProposerPreparationDto { - validator_index: entry.index, - fee_recipient: crate::beacon_node::dto::encode_hex(&fee_recipient.0), - }) - }) - .collect(); - if !preparations.is_empty() - && let Err(err) = beacon_node.prepare_beacon_proposer(&preparations).await - { + // duties, which is worse and unrelated. The duty loop sends again once + // this slot's duties are done. + let sent = prepare_proposers(beacon_node.as_ref(), &entries, settings).await; + let _ = sent.inspect_err(|err| { warn!( %epoch, %err, - "Failed to register this client's fee recipients; blocks proposed this epoch may \ - pay somewhere else" + "Failed to register this client's fee recipients; a block proposed before the \ + re-send after this slot's duties may pay somewhere else" ); - } + }); let changed = duties.refresh_around(epoch).await?; refresh_ptc(duties, epoch, context).await; @@ -1045,9 +1053,181 @@ async fn refresh_epoch( Ok(Refreshed { moved: changed, registrable: registrable.map(|entry| entry.pubkey).collect(), + resolved: entries, }) } +/// Tell the beacon node the address each of `entries`' blocks pays: one +/// `prepare_beacon_proposer` call, or none when no validator has an address. +/// The epoch refresh's send; the duty loop's per-slot one is [`Preparations`]. +async fn prepare_proposers( + beacon_node: &B, + entries: &[ValidatorEntry], + settings: &ProposerSettings, +) -> Result<()> { + let preparations = preparations(entries, settings); + if preparations.is_empty() { + return Ok(()); + } + beacon_node.prepare_beacon_proposer(&preparations).await +} + +/// One preparation for each of `entries` that has an address, its own or the +/// default, as `settings` names it now. +/// +/// A validator with neither is left out: there is nothing to tell the node, +/// which then picks an address itself, as the startup warning says. +fn preparations<'a>( + entries: impl IntoIterator, + settings: &ProposerSettings, +) -> Vec { + entries + .into_iter() + .filter_map(|entry| { + let fee_recipient = settings.fee_recipient(&entry.pubkey)?; + Some(ProposerPreparationDto { + validator_index: entry.index, + fee_recipient: crate::beacon_node::dto::encode_hex(&fee_recipient.0), + }) + }) + .collect() +} + +/// The proposer preparations, sent again once every slot's duties are done. +/// +/// # Every slot, not once an epoch +/// +/// As Lighthouse's validator client does. A node keeps a preparation for the +/// epoch it arrived in and two more, and loses all of them when it restarts; +/// until the next one arrives it builds its own payloads for the zero address. +/// Sent only by the epoch refresh, a restart mid-epoch cost the rest of that +/// epoch's local blocks their execution-layer rewards: six blocks on a devnet, +/// from a beacon node killed 14 slots into its epoch. Sent every slot, a +/// restart costs at most a proposal that comes before the first re-send the +/// node is back up for, which is within a slot of its return. +/// +/// # What is sent +/// +/// The validators the last refresh resolved that the store still holds, each +/// with the address [`ProposerSettings`] gives it at the send, so a +/// keymanager change to a fee recipient reaches the node after the duties of +/// the slot it is made in, or the next slot's while a slow node still holds +/// the last send (see below). Three cases that does not cover: +/// +/// - **A key deleted through the keymanager** (`DELETE /eth/v1/keystores` or +/// `/remotekeys`) is not sent again. Its overrides are forgotten with it, so +/// it would otherwise go out with `--suggested-fee-recipient`, overwriting +/// the preparation of a client the key moved to on the same node. The node +/// keeps the address it was last given until it expires it or restarts. +/// Imported again before the next refresh, the key is sent again, with the +/// defaults a key starts on once its overrides are forgotten. +/// - **A `DELETE` of a validator's fee recipient that leaves it no address** +/// (no `--suggested-fee-recipient` either) sends nothing for it, so the node +/// keeps the address it was last given, the same way. +/// - **A key imported mid-epoch** is not sent until the next refresh resolves +/// its index. Nothing is lost: that refresh is also what fetches its +/// proposer duties (they are narrowed to the indices it resolves, see +/// [`DutiesService::set_indices`]), so this client proposes no block for it +/// before the refresh that prepares it. +/// +/// # One send at a time +/// +/// Each send reads the settings, then waits on every node. Against a slow +/// node a send can outlive the next slot's, and two in flight could land in +/// either order, reverting a fee recipient changed in between. So a send that +/// finds the last one still running skips, and the slot after it sends what +/// is current then (see [`crate::in_flight`]). A node that never answers is +/// cut off by the HTTP client's request timeout, 8 s, so a hung node costs +/// the healthy ones only the re-sends that come due while a send waits on it: +/// at most every other one on 12 s slots, more on shorter ones, never all of +/// them. +/// +/// The epoch refresh's send does not take part. It runs on the duty loop +/// ahead of the epoch's first proposal, where skipping would cost that +/// proposal its preparation, and waiting on a hung node would cost it up to +/// that timeout. A re-send still in flight from the slot before can then land +/// after it, which is why the refresh's slot re-sends too, once its duties are +/// done, although its refresh just sent the same: the late send can then cost +/// at most the epoch's first proposal. If it is still running at that point, +/// that re-send skips and the next slot's puts the current address back. +/// +/// # Quiet on failure +/// +/// A node that is down, restarting or refusing costs a `debug!` line, here +/// and per node (see [`BeaconNodeApi::prepare_beacon_proposer_quietly`]): the +/// slot's duties already report a node that is down, and the next slot tries +/// again. The refresh's send still warns. +struct Preparations { + beacon_node: Arc, + settings: Arc, + /// The re-send, while it runs. + in_flight: InFlight, +} + +impl Preparations { + fn new(beacon_node: Arc, settings: Arc) -> Self { + Self { + beacon_node, + settings, + in_flight: InFlight::default(), + } + } + + /// The duty loop's gate, once every slot's duties are done, the + /// refresh's slot included: spawn the re-send of `resolved`'s + /// preparations, unless there is nobody to prepare. Returns the spawned + /// task's handle so a test can await it; the duty loop drops it, as it + /// does its other sends' handles. + fn after_slot( + self: &Arc, + slot: u64, + resolved: &Arc>, + store: &Arc>, + ) -> Option> { + if resolved.is_empty() { + return None; + } + let resend = self.clone().resend(slot, resolved.clone(), store.clone()); + Some(tokio::spawn(resend)) + } + + async fn resend( + self: Arc, + slot: u64, + resolved: Arc>, + store: Arc>, + ) { + // Before the settings are read, so the sends reach the nodes in the + // order they read them. + let Some(_running) = self.in_flight.enter() else { + debug!( + %slot, + "The last fee recipient re-send is still running; skipping this slot's" + ); + return; + }; + // Built under the store's read guard, which is released before the + // send, so a keymanager write waits on no node. + let preparations = { + let store = store.read().await; + let held = resolved + .iter() + .filter(|entry| store.contains(&entry.pubkey)); + preparations(held, &self.settings) + }; + if preparations.is_empty() { + return; + } + let sent = self + .beacon_node + .prepare_beacon_proposer_quietly(&preparations) + .await; + let _ = sent.inspect_err(|err| { + debug!(%slot, %err, "Failed to re-send this client's fee recipients"); + }); + } +} + /// Whether a validator whose Beacon API `status` is this one is sent a builder /// registration: one that is pending or active, not one that exited or was /// withdrawn. @@ -1206,10 +1386,12 @@ mod tests { .expect("refreshes"); } - /// One preparation per resolved validator, every epoch. The node keeps a - /// preparation for three epochs and forgets all of them on restart, so a - /// client that sent this once at startup would quietly stop being - /// registered. + /// One preparation per resolved validator from every refresh, ahead of + /// the epoch's duties. The node keeps a preparation for three epochs and + /// forgets all of them on restart, so a client that sent this once at + /// startup would quietly stop being registered; the duty loop also sends + /// them again after every slot's duties (see + /// `the_preparations_go_out_again_after_every_slot_the_refreshs_included`). #[tokio::test] async fn every_resolved_validator_is_registered_for_its_fee_recipient() { let node = Arc::new(node()); @@ -1235,6 +1417,147 @@ mod tests { assert_eq!(node.preparations().len(), 4); } + /// A store holding `keys`, behind a signer nothing calls: these tests ask + /// the store which keys it holds, never for a signature. + fn store_holding(keys: &[BlsPubkey]) -> Arc> { + let mut store = ValidatorStore::new(); + for key in keys { + store + .insert_remote(*key, "http://127.0.0.1:9000") + .expect("inserts"); + } + Arc::new(RwLock::new(store)) + } + + /// Every preparation `node` was sent, as (index, address), in order. + fn sent(node: &MockBeaconNode) -> Vec<(u64, String)> { + node.preparations() + .into_iter() + .map(|entry| (entry.validator_index, entry.fee_recipient)) + .collect() + } + + /// The address whose twenty bytes are all `byte`, as a preparation + /// carries it. + fn address(byte: u8) -> String { + crate::beacon_node::dto::encode_hex(&[byte; 20]) + } + + /// The duty loop's gate: after every slot's duties, the refresh's + /// included, the refresh's validators again, with the addresses the + /// settings name by then. That re-send is what a beacon node restarted + /// mid-epoch hears from first: sent only by the refresh, its local blocks + /// paid the zero address for the rest of the epoch. + /// + /// On the refresh's own slot it puts the current address back when the + /// refresh's send was overtaken, here by the previous slot's re-send + /// landing late with the address a keymanager change replaced. Skipped + /// there, that address would stand until after the next slot's duties, + /// past the next proposal. + #[tokio::test] + async fn the_preparations_go_out_again_after_every_slot_the_refreshs_included() { + let node = Arc::new(node()); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + let settings = Arc::new(settings(Some(H160([0xab; 20])))); + settings.set_fee_recipient(&pubkey(2), H160([0xcd; 20])); + let keys = [pubkey(1), pubkey(2)]; + let store = store_holding(&keys); + let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); + + let Refreshed { resolved, .. } = + refresh_epoch(&node, &mut duties, &keys, 3, &settings, &context()) + .await + .expect("refreshes"); + assert_eq!(sent(&node), [(11, address(0xab)), (22, address(0xcd))]); + // The previous slot's re-send, which read validator 22's address + // before the change, reaching the node after the refresh's send. + let late = ProposerPreparationDto { + validator_index: 22, + fee_recipient: address(0xab), + }; + node.preparations.lock().expect("lock").push(late); + + let resolved = Arc::new(resolved); + preparations + .after_slot(96, &resolved, &store) + .expect("the refresh's slot re-sends too") + .await + .expect("the re-send ends"); + assert_eq!(sent(&node)[3..], [(11, address(0xab)), (22, address(0xcd))]); + + settings.set_fee_recipient(&pubkey(2), H160([0xef; 20])); + preparations + .after_slot(97, &resolved, &store) + .expect("and so does every slot after it") + .await + .expect("the re-send ends"); + assert_eq!(sent(&node)[5..], [(11, address(0xab)), (22, address(0xef))]); + } + + /// A key the keymanager deleted after the refresh is not prepared again. + /// Its overrides went with it, so it would otherwise go out every slot + /// with the default address, overwriting the preparation of a client the + /// key moved to on the same node. + #[tokio::test] + async fn a_key_deleted_since_the_refresh_is_not_prepared_again() { + let node = Arc::new(node()); + let settings = Arc::new(settings(Some(H160([0xab; 20])))); + settings.set_fee_recipient(&pubkey(2), H160([0xcd; 20])); + let store = store_holding(&[pubkey(1), pubkey(2)]); + let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); + let resolved = Arc::new(node.validators.clone()); + + // What a keystore or remote key `DELETE` does to the two. + store.write().await.remove(&pubkey(2)); + settings.forget(&pubkey(2)); + preparations + .after_slot(97, &resolved, &store) + .expect("due") + .await + .expect("the re-send ends"); + + assert_eq!(sent(&node), [(11, address(0xab))]); + } + + /// One re-send at a time. A slot whose re-send finds the last one still + /// waiting on a node skips, so a send that read an older fee recipient + /// cannot land after one that read the newer; the slot after sends what + /// is current then. + #[tokio::test] + async fn a_re_send_still_waiting_on_a_node_makes_the_next_slot_skip() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let node = Arc::new(node().with_preparation_gate(gate.clone())); + let settings = Arc::new(settings(Some(H160([0xab; 20])))); + let store = store_holding(&[pubkey(1), pubkey(2)]); + let preparations = Arc::new(Preparations::new(node.clone(), settings.clone())); + let resolved = Arc::new(node.validators.clone()); + + let slow = preparations.after_slot(97, &resolved, &store).expect("due"); + node.preparation_entered.notified().await; + settings.set_fee_recipient(&pubkey(1), H160([0xcd; 20])); + let skipped = preparations.after_slot(98, &resolved, &store).expect("due"); + tokio::time::timeout(std::time::Duration::from_secs(5), skipped) + .await + .expect("a skipped re-send does not wait on the node") + .expect("the skipped re-send ends"); + assert!( + node.preparations().is_empty(), + "the slow re-send is still parked at the gate, and the skipped one sent nothing" + ); + + gate.add_permits(1); + slow.await.expect("the slow re-send ends"); + assert_eq!(sent(&node), [(11, address(0xab)), (22, address(0xab))]); + + gate.add_permits(1); + preparations + .after_slot(99, &resolved, &store) + .expect("due") + .await + .expect("the re-send ends"); + assert_eq!(sent(&node)[2..], [(11, address(0xcd)), (22, address(0xab))]); + } + /// The refresh says when the attester schedule moved, which is what owes /// the node its subnet subscriptions, and sends none itself: they wait /// for the slot's block, since each costs a remote key a round trip to diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs index 250da6a26..0fa9d5574 100644 --- a/crates/validator/src/preferences.rs +++ b/crates/validator/src/preferences.rs @@ -58,11 +58,15 @@ //! A node keeps the preferences it accepted in memory only, so one that //! restarts has lost them, and gossip does not give them back. Every kept //! preference whose slot is still ahead is therefore sent again, as signed, -//! once an epoch, as the duty loop re-sends `prepare_beacon_proposer`, which a -//! restart loses too. The round runs at the poll of an epoch's second-to-last -//! slot, or at the next poll after it when that one was skipped behind a slow -//! node, and re-sends each kept preference whose slot has not started once -//! the poll's duties are in. Normally that is the epoch's last slot and every +//! once an epoch. `prepare_beacon_proposer`, which a restart loses too, is +//! re-sent every slot instead (see the duty loop's `Preparations`): that is +//! one request for every validator, where the kept preferences are one +//! request each (see "One preference per request" below). +//! +//! The round runs at the poll of an epoch's second-to-last slot, or at the +//! next poll after it when that one was skipped behind a slow node, and +//! re-sends each kept preference whose slot has not started once the poll's +//! duties are in. Normally that is the epoch's last slot and every //! slot of the next one, so a node restarted mid-epoch lacks the preferences //! of the slots before the round and holds every later one. One restarted //! after the round lacks the next epoch's too, all but its last slot's, which diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index e62e2b16c..f2f0f5998 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -323,9 +323,10 @@ impl ProposalService { "Block pays its execution-layer rewards to an address this client did not ask for; \ signing it anyway, since refusing would also forfeit the consensus reward and cost \ the network a slot. Check this beacon node's proposer preparation. A fee recipient \ - changed through the keymanager API reaches the node at the next epoch boundary; \ - from gloas, a proposer preference published before the change (by this process \ - before a restart, say) keeps the old address until the next dependent root." + changed through the keymanager API reaches the node after the duties of the slot \ + it was changed in; from gloas, a proposer preference published before the change \ + (by this process before a restart, say) keeps the old address until the next \ + dependent root." ); crate::metrics::inc_fee_recipient_mismatches(); } diff --git a/crates/validator/src/proposer_settings.rs b/crates/validator/src/proposer_settings.rs index 31e30321b..1abb2853c 100644 --- a/crates/validator/src/proposer_settings.rs +++ b/crates/validator/src/proposer_settings.rs @@ -24,10 +24,15 @@ //! //! Graffiti and gas limit are read when they are used: the next proposal //! carries the new value. A fee recipient reaches the beacon node through -//! `prepare_beacon_proposer`, which the duty loop sends once per epoch, so a -//! change is registered at the next epoch boundary. A block proposed before -//! then pays the address the node was last told, which the check before -//! signing reports as a mismatch. +//! `prepare_beacon_proposer`, which the duty loop sends again once every +//! slot's duties are done, so a change is registered after the duties of the +//! slot it is made in (the next slot's while a slow node holds the last +//! send). That is for a validator the last epoch refresh resolved; one +//! imported since is first sent once the next refresh resolves its index. A +//! block proposed before then pays the address the node was last told, which +//! the check before signing reports as a mismatch. A `DELETE` that leaves a +//! validator no address at all sends nothing for it, so the node keeps the +//! old one until it expires it or restarts. //! //! From gloas a node pays the address in the proposer preferences it holds for //! the slot instead, and gas limit likewise, and a preference cannot be diff --git a/docs/cli.md b/docs/cli.md index b04b4d89e..91ee95a10 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -305,7 +305,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--http-address` | `127.0.0.1` | Bind address for the metrics and keymanager servers | | `--metrics-port` | `5064` | Prometheus metrics port | | `--keymanager-port` | `5062` | Keymanager API port. Only bound with `--enable-keymanager` | -| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours. The zero address is accepted, as every other client accepts it, but reported at error at startup: it burns the rewards, and builder registrations and proposer preferences sign it for the builder to pay | +| `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours. The addresses reach each beacon node as one `prepare_beacon_proposer` request every slot, listing each validator that has an index on chain and an address, as Lighthouse's validator client sends it. The zero address is accepted, as every other client accepts it, but reported at error at startup: it burns the rewards, and builder registrations and proposer preferences sign it for the builder to pay | | `--gas-limit` | `60000000` | The gas limit every validator targets, unless the keymanager API gives it its own. Before gloas it goes into builder registrations (with `--builder-proposals`). From gloas it goes into the proposer preferences, which are published for every proposal, and the beacon node builds this client's own payloads toward it as well, so a network whose gas limit is not 60M needs it set, or every proposal pulls the limit toward 60M. **Sepolia and PlatÃ¥berget schedule `200000000`** (their `config.yaml`'s `GAS_LIMIT_SCHEDULE`, which this client does not read yet, [#795](https://github.com/lambdaclass/ethlambda/issues/795)), so on either pass `--gas-limit 200000000`. Must be positive | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, remote keys, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | @@ -615,6 +615,14 @@ entry and leaves the local key, and its per-validator settings, as they are. Its answer is `error` too, naming `/eth/v1/keystores`, and never `deleted`: the key is still signing here until that route removes it. +A deleted key's fee recipient is no longer sent to the beacon node, from the +next slot on. The node keeps the address it was last given for that validator +until it expires the entry or restarts, so a client the key moved to on the +same node prepares it with its own address without this one overwriting it. +An imported key's fee recipient is first sent at the next epoch boundary, when +its index is resolved, which is also when its proposer duties are first +fetched. + It also serves `POST /eth/v1/validator/{pubkey}/voluntary_exit`, which signs a voluntary exit for a validator this client holds, in a local keystore or in a remote signer, and answers with the `SignedVoluntaryExit`, for the exit's @@ -630,7 +638,7 @@ give one validator its own value in place of the command line's default: | Setting | Default | Takes effect | |---|---|---| -| `feerecipient` | `--suggested-fee-recipient` | At the next epoch boundary, when the fee recipients are registered with the beacon node again. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no registration, so the builder keeps paying the address it was last given. From gloas, the proposer preferences carry it, from the next dependent root on | +| `feerecipient` | `--suggested-fee-recipient` | Once the duties of the slot it is set in are done, when the fee recipients are sent to the beacon node again, as they are after every slot's duties (the next slot's while a slow beacon node still holds the last send). A validator imported since the last epoch boundary, or with no index on chain yet, is first sent once an epoch boundary resolves its index. A block proposed before then pays the address the node was last given, and the check before signing reports the mismatch. With `--builder-proposals`, the next builder registration carries it too, usually at the next slot (later while the epoch's full registration send is still running) and again every slot until a node accepts it or the epoch's full send carries it, and an ethlambda beacon node takes it from there. A `DELETE` that leaves a validator with no address at all (no `--suggested-fee-recipient` either) sends no fee recipient and no registration, so the beacon node and the builder keep paying the address they were last given. From gloas, the proposer preferences carry it, from the next dependent root on | | `graffiti` | `--graffiti` | The next proposal | | `gas_limit` | `--gas-limit` | Before gloas, the next builder registration with `--builder-proposals` (usually at the next slot, later while the epoch's full registration send is still running, and again every slot until a node accepts it or the epoch's full send carries it), and stored and served only without it. From gloas, the proposer preferences' target gas limit, from the next dependent root on, flag or not | diff --git a/docs/rpc.md b/docs/rpc.md index c26c5b0f4..9a530a3c6 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -412,7 +412,9 @@ the chain actor writes, so no request waits on the actor. next slot's block is asked for. Entries more than an epoch old are dropped once a slot. - **`prepare_beacon_proposer`** records each validator's fee recipient, in - memory (a validator client repeats the call every epoch). + memory, so a restart forgets it and the node builds its own payloads for the + zero address until the next call. ethlambda's validator client repeats the + call every slot, as Lighthouse's does. - **`register_validator`** takes a JSON array or an SSZ list of signed builder registrations. Registrations of validators the head state does not hold, or holds as exited, are dropped and counted diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index b36d39c75..948dda188 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -839,4 +839,7 @@ signature, whichever way the node answered. Three departures: node holds them in memory only and gossip does not replay them, so a node that restarts would otherwise self-build without them, and ignore builders' bids, until the next dependent - root. Same cadence as `prepare_beacon_proposer`, which a restart loses too. + root. `prepare_beacon_proposer`, which a restart loses too, is re-sent every + slot instead: it is one request for every validator, where the kept + preferences are one request each, since a node answers a call carrying + several with one status for all of them.