From 74f572b5203684622cad1eeee19650728af7f770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:38:55 -0300 Subject: [PATCH 1/2] fix(beacon): give the gloas fork epoch's PTC duties their own dependent root The payload timeliness committee of GLOAS_FORK_EPOCH is computed at the fork, by upgrade_to_gloas, from the effective balances the last pre-gloas epoch ends with. beacon-APIs (apis/validator/duties/ptc.yaml) therefore gives that epoch its own dependent root, the block root at compute_start_slot_at_epoch(epoch) - 1 (the genesis block root on underflow), and says its duties are not available before the fork. The node used the generic rule, compute_start_slot_at_epoch(epoch - 1) - 1, for every epoch, and asked about the fork epoch during the epoch before it, served a committee from the pre-gloas head advanced to the fork, missing every block still to come in that epoch. The validator client fetched it then as lookahead, and at the fork epoch's first slot fetched it again, but DutiesService keeps a held schedule while its dependent root is unchanged. The wrong root never changed, so the client kept the stale early committee for the whole first gloas epoch. Node: the fork epoch's dependent slot is now its start slot minus one, and the epoch is refused with a 400 until its first slot has started by the wall clock, allowing MAXIMUM_GOSSIP_CLOCK_DISPARITY, while the head is still before it (so a chain with gloas at genesis is still served before genesis). Pre-gloas epochs still answer 200 with no duties. Validator client: the fork epoch's duties are no longer fetched as the previous epoch's lookahead, which against Lighthouse drew a 400 and a misleading "may miss its votes" warning. They are fetched at the fork epoch's first slot, with the next epoch as the lookahead. Lighthouse does the same: it refuses the fork epoch until the tolerant current epoch reaches it, and takes its dependent root from the block at the fork's start slot minus one. --- crates/net/rpc/src/beacon/ptc.rs | 209 +++++++++++++++++++++-- crates/validator/src/beacon_node/mock.rs | 12 +- crates/validator/src/duties.rs | 6 +- crates/validator/src/lib.rs | 51 +++--- docs/cli.md | 5 +- docs/rpc.md | 10 +- 6 files changed, 249 insertions(+), 44 deletions(-) diff --git a/crates/net/rpc/src/beacon/ptc.rs b/crates/net/rpc/src/beacon/ptc.rs index 1a00631e4..b4379a62c 100644 --- a/crates/net/rpc/src/beacon/ptc.rs +++ b/crates/net/rpc/src/beacon/ptc.rs @@ -28,7 +28,7 @@ use ethlambda_state_transition::beacon::{ fork_choice::get_payload_due_ms, gloas_block_production::aggregate_payload_attestations, gossip::{ - Outcome, + Outcome, is_future_slot, payload_attestation::{SeenPayloadAttestations, cheap_checks, stateful_checks}, }, helpers::{accessors::get_block_root_at_slot, gloas::get_ptc_assignments}, @@ -110,12 +110,23 @@ struct PtcDuty { /// /// An epoch before gloas has no committee, so the answer is empty rather than /// an error: a validator client asks every epoch and reads the empty list as -/// "no duty". +/// "no duty". This stays a `200`, not the `400` the fork epoch gets below. +/// +/// The gloas fork epoch itself is refused with a `400` until its first slot +/// has started by the wall clock, allowing `MAXIMUM_GOSSIP_CLOCK_DISPARITY` +/// (as Lighthouse does), while the head is still before it. Its committee is +/// computed at the fork, by `upgrade_to_gloas`, from the effective balances the +/// last pre-gloas epoch ends with, so until then any answer is one the rest of +/// that epoch can change (beacon-APIs `ptc.yaml`: "Duties for this epoch are +/// not available before the fork"). /// /// `dependent_root` is the block root at `compute_start_slot_at_epoch(epoch - 1) - 1` /// (the genesis block's at the start), the same definition the attester duties /// use: the committee window is fixed by the state at the end of the epoch -/// before the previous one. +/// before the previous one. The fork epoch is the exception, for the reason +/// above: its root is the block at `compute_start_slot_at_epoch(epoch) - 1`, +/// the last slot before the fork (the genesis block's when gloas starts at +/// genesis). async fn post_ptc_duties( Path(epoch): Path, State(store): State, @@ -128,7 +139,11 @@ async fn post_ptc_duties( Ok(wanted) => wanted, Err(err) => return err.into_response(), }; - let computed = tokio::task::spawn_blocking(move || ptc_duties(&store, epoch, &wanted)).await; + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis() as u64); + let computed = + tokio::task::spawn_blocking(move || ptc_duties(&store, epoch, &wanted, now_ms)).await; match computed { Ok(Ok(body)) => crate::json_response(body), Ok(Err(err)) => err.into_response(), @@ -149,10 +164,13 @@ fn parse_validator_indices(body: &[u8]) -> Result, ApiEr .map_err(|_| ApiError::BadRequest("invalid validator index")) } +/// The duties of [`post_ptc_duties`], with `now_ms` the wall clock the gloas +/// fork epoch's refusal is judged against. fn ptc_duties( store: &Store, epoch: Epoch, wanted: &HashSet, + now_ms: u64, ) -> Result { let config = store.config(); let (head_root, head_state) = head(store)?; @@ -167,7 +185,26 @@ fn ptc_duties( )); } - let dependent_slot = compute_start_slot_at_epoch(epoch.saturating_sub(1)).saturating_sub(1); + // The fork epoch's committee does not exist until the fork, so it is not + // served before then (see the handler). Only while the head is before the + // fork, which leaves a chain with gloas at genesis answerable before + // genesis: its committee is in the genesis state already. + let fork_epoch = epoch == config.gloas_fork_epoch; + let start_slot = compute_start_slot_at_epoch(epoch); + if fork_epoch && state_epoch < epoch && is_future_slot(&config, start_slot, now_ms) { + return Err(ApiError::BadRequest( + "PTC duties for the gloas fork epoch are not available until the fork", + )); + } + + // The fork epoch's committee is fixed by the last state before the fork, + // every other epoch's by the end of the epoch before the previous one. Both + // saturate to the genesis block's slot. + let dependent_slot = if fork_epoch { + start_slot.saturating_sub(1) + } else { + compute_start_slot_at_epoch(epoch.saturating_sub(1)).saturating_sub(1) + }; let dependent_root = if dependent_slot >= head_state.slot() { head_root } else { @@ -182,6 +219,8 @@ fn ptc_duties( }) }; + // Before gloas there is no committee: an empty `200`, not an error, so a + // client can ask every epoch. if epoch < config.gloas_fork_epoch { return Ok(respond(Vec::new())); } @@ -196,7 +235,6 @@ fn ptc_duties( let state: Arc = if readable_from_head { head_state } else { - let start_slot = compute_start_slot_at_epoch(epoch); let mut advanced = (*head_state).clone(); process_slots(&mut advanced, start_slot, &config) .map_err(|_| ApiError::Internal("advancing the head state failed"))?; @@ -825,22 +863,15 @@ mod tests { } /// While the head is still fulu, the first gloas epoch is answered from a - /// copy of the head state advanced across the upgrade. + /// copy of the head state advanced across the upgrade. The store's clock is + /// mainnet's, long past the fork, so the fork epoch is not refused. #[tokio::test] async fn the_first_gloas_epoch_is_computed_from_a_fulu_head() { - use ethlambda_state_transition::beacon::helpers::fulu::initialize_proposer_lookahead; - let mut state = with_signing_validators_at(ForkName::Fulu, COUNT); - let lookahead = initialize_proposer_lookahead(&state).unwrap(); - let BeaconState::Fulu(fulu) = &mut state else { - unreachable!("built as fulu") - }; - fulu.proposer_lookahead = lookahead.try_into().unwrap(); + let state = fulu_state(); let head_epoch = compute_epoch_at_slot(state.slot()); let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, head_epoch + 1); let (store, _root) = crate::test_utils::beacon_store_with_config(state, config); - let everyone = - serde_json::to_string(&(0..COUNT as u64).map(|i| i.to_string()).collect::>()) - .unwrap(); + let everyone = everyone(); let reply = request(store.clone(), duties_request(head_epoch + 1, &everyone)).await; @@ -860,6 +891,150 @@ mod tests { assert_eq!(reply.json()["data"], serde_json::json!([])); } + /// A fulu state with a real proposer lookahead (epoch processing needs it + /// to cross the gloas upgrade) and distinct `block_roots`. + fn fulu_state() -> BeaconState { + use ethlambda_state_transition::beacon::helpers::fulu::initialize_proposer_lookahead; + let mut state = with_signing_validators_at(ForkName::Fulu, COUNT); + let lookahead = initialize_proposer_lookahead(&state).unwrap(); + let BeaconState::Fulu(fulu) = &mut state else { + unreachable!("built as fulu") + }; + fulu.proposer_lookahead = lookahead.try_into().unwrap(); + for slot in 0..fulu.block_roots.len() { + fulu.block_roots[slot] = slot_root(slot as u64); + } + state + } + + /// The fork epoch used by the boundary tests below. + const FORK_EPOCH: Epoch = 2; + + /// A fulu head at the last slot before gloas at [`FORK_EPOCH`], on a clock + /// whose slot `clock_slot` began a second ago. + fn fulu_head_before_the_fork(clock_slot: Slot) -> (Store, H256) { + let mut state = fulu_state(); + let BeaconState::Fulu(fulu) = &mut state else { + unreachable!("built as fulu") + }; + fulu.slot = compute_start_slot_at_epoch(FORK_EPOCH) - 1; + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, FORK_EPOCH); + store_with_head_at_clock(state, config, 0, clock_slot) + } + + fn everyone() -> String { + serde_json::to_string(&(0..COUNT as u64).map(|i| i.to_string()).collect::>()) + .unwrap() + } + + /// The fork epoch's committee is fixed by the last state before the fork, + /// so its `dependent_root` is the block at the fork's first slot minus one, + /// not the generic rule's block an epoch earlier. The epoch after it keeps + /// the generic rule, which names the same slot. + #[tokio::test] + async fn the_fork_epochs_dependent_root_is_the_last_block_before_the_fork() { + let state = gloas_state(); + let fork_epoch = compute_epoch_at_slot(state.slot()); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, fork_epoch); + let (store, _) = store_with_head(state, config, 0); + let last_before_fork = compute_start_slot_at_epoch(fork_epoch) - 1; + + for epoch in [fork_epoch, fork_epoch + 1] { + let reply = request(store.clone(), duties_request(epoch, r#"["1"]"#)).await; + assert_eq!(reply.status, StatusCode::OK, "epoch {epoch}"); + assert_eq!( + reply.json()["dependent_root"], + format!("{:?}", slot_root(last_before_fork)), + "epoch {epoch}" + ); + } + // The generic rule would have named the block an epoch earlier. + assert_ne!( + compute_start_slot_at_epoch(fork_epoch - 1).saturating_sub(1), + last_before_fork + ); + + // A pre-gloas epoch is untouched: still an empty `200` on the generic + // rule's root. + let reply = request(store, duties_request(fork_epoch - 1, r#"["1"]"#)).await; + assert_eq!(reply.status, StatusCode::OK); + assert_eq!(reply.json()["data"], serde_json::json!([])); + assert_eq!( + reply.json()["dependent_root"], + format!("{:?}", slot_root(0)) + ); + } + + /// Before the fork the fork epoch's committee does not exist yet, so it is + /// refused; from the boundary on it is served, from the head state advanced + /// across the upgrade, and depends on the head: the last block before the + /// fork. + #[tokio::test] + async fn the_fork_epoch_is_refused_before_the_fork_and_served_from_the_boundary() { + let fork_start = compute_start_slot_at_epoch(FORK_EPOCH); + + let (store, _) = fulu_head_before_the_fork(fork_start - 1); + let reply = request(store.clone(), duties_request(FORK_EPOCH, &everyone())).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + // The epoch before the fork is still an empty `200`. + let reply = request(store, duties_request(FORK_EPOCH - 1, &everyone())).await; + assert_eq!(reply.status, StatusCode::OK); + assert_eq!(reply.json()["data"], serde_json::json!([])); + + let (store, head_root) = fulu_head_before_the_fork(fork_start); + let reply = request(store, duties_request(FORK_EPOCH, &everyone())).await; + assert_eq!(reply.status, StatusCode::OK, "{:?}", reply.body); + let json = reply.json(); + assert_eq!(json["dependent_root"], format!("{head_root:?}")); + let duties = json["data"].as_array().unwrap(); + assert!( + !duties.is_empty(), + "the committee seats some of the registry" + ); + for duty in duties { + let slot: u64 = duty["slot"].as_str().unwrap().parse().unwrap(); + assert!((fork_start..fork_start + preset::SLOTS_PER_EPOCH).contains(&slot)); + } + } + + /// The refusal allows gossip's clock disparity: a request that far ahead of + /// the fork's first slot is served, one a millisecond earlier is not. + #[test] + fn the_fork_epoch_is_served_within_the_clock_disparity() { + use ethlambda_types::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY; + let fork_start = compute_start_slot_at_epoch(FORK_EPOCH); + let (store, _) = fulu_head_before_the_fork(fork_start - 1); + let config = store.config(); + let fork_start_ms = config.genesis_time_ms() + fork_start * config.slot_duration_ms; + let wanted = HashSet::from([1]); + + let earliest = fork_start_ms - MAXIMUM_GOSSIP_CLOCK_DISPARITY; + assert!(ptc_duties(&store, FORK_EPOCH, &wanted, earliest).is_ok()); + let too_early = ptc_duties(&store, FORK_EPOCH, &wanted, earliest - 1); + assert!(matches!(too_early, Err(ApiError::BadRequest(_)))); + } + + /// With gloas at genesis the fork epoch's committee is in the genesis + /// state, so it is served before genesis too, as it was before the fork + /// epoch was refused at all. + #[test] + fn a_chain_with_gloas_at_genesis_serves_its_first_epoch_before_genesis() { + let mut state = gloas_state(); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + inner.slot = 0; + let (store, head_root) = store_with_head_at_clock(state, gloas_config(), 0, 0); + let before_genesis = store.config().genesis_time_ms() - 60_000; + + let answer = ptc_duties(&store, 0, &HashSet::from([1]), before_genesis); + + let Ok(json) = answer else { + panic!("refused: {answer:?}") + }; + assert_eq!(json["dependent_root"], format!("{head_root:?}")); + } + // ----- payload attestation data ---------------------------------------- fn data_request(slot: u64, accept: Option<&str>) -> Request { diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 79b9a70d6..08eae13b6 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -184,7 +184,8 @@ pub struct MockBeaconNode { /// Every published gloas aggregate batch. pub published_gloas_aggregates: Mutex)>>, pub ptc: Mutex>, - pub ptc_calls: Mutex, + /// The epoch of every `ptc_duties` request, in call order. + pub ptc_requests: Mutex>, /// What `payload_attestation_data` answers with. `None` is the 204. pub payload_attestation_data: Option, pub payload_attestation_data_calls: Mutex, @@ -594,7 +595,12 @@ impl MockBeaconNode { } pub fn ptc_call_count(&self) -> usize { - *self.ptc_calls.lock().expect("lock") + self.ptc_requests.lock().expect("lock").len() + } + + /// The epochs `ptc_duties` was asked for, in call order. + pub fn ptc_requested_epochs(&self) -> Vec { + self.ptc_requests.lock().expect("lock").clone() } /// Answer `payload_attestation_data` with `data`. @@ -1100,7 +1106,7 @@ impl BeaconNodeApi for MockBeaconNode { async fn ptc_duties(&self, epoch: Epoch, _indices: &[ValidatorIndex]) -> Result { self.guard("ptc_duties")?; - *self.ptc_calls.lock().expect("lock") += 1; + self.ptc_requests.lock().expect("lock").push(epoch); self.ptc .lock() .expect("lock") diff --git a/crates/validator/src/duties.rs b/crates/validator/src/duties.rs index b1073811a..cd8b3372b 100644 --- a/crates/validator/src/duties.rs +++ b/crates/validator/src/duties.rs @@ -25,7 +25,8 @@ //! next, replaced when the `dependent_root` changes. They are narrowed to this //! client's indices on arrival, and are not part of subnet subscriptions //! either. Whether an epoch is a gloas one is the caller's to decide, since -//! this service holds no fork schedule. +//! this service holds no fork schedule, and so is holding off the gloas fork +//! epoch's until the fork, before which a node cannot know them. use std::collections::HashMap; use std::sync::Arc; @@ -224,7 +225,8 @@ impl DutiesService { /// changed. /// /// Only to be called for a gloas epoch: an earlier one answers with no - /// duties, which is harmless but a request wasted every epoch. + /// duties, which is harmless but a request wasted every epoch. Nor for the + /// gloas fork epoch before it has begun, which a node refuses. pub async fn refresh_ptc(&mut self, epoch: Epoch) -> Result<()> { if self.indices.is_empty() { return Ok(()); diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index d368862d3..f579ad8a3 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -1243,11 +1243,15 @@ fn is_registrable(status: &str) -> bool { } /// Fetch the payload timeliness committee schedule for this epoch and the next, -/// each only if it is a gloas epoch. +/// each only if it is a gloas epoch, and the gloas fork epoch's only once it +/// has begun. /// /// Gated per epoch rather than once, because the fork can activate between the -/// two: the lookahead is the epoch the first gloas slots belong to when the -/// current one is the last before the fork. A failed fetch is logged and +/// two. The fork epoch is no lookahead, though: its committee is computed at +/// the fork from the effective balances the last pre-gloas epoch ends with, so +/// a node serves it only from the fork on (beacon-APIs `ptc.yaml`; Lighthouse +/// answers `400` before). It is fetched at the fork epoch's own first slot, +/// with the epoch after it as the lookahead. A failed fetch is logged and /// shrugged off, like the proposer and lookahead ones: it must not cost this /// epoch's attester schedule, which is already in hand. async fn refresh_ptc( @@ -1259,6 +1263,9 @@ async fn refresh_ptc( if context.config.fork_at_epoch(target) < ethlambda_types::beacon::fork::ForkName::Gloas { continue; } + if target == context.config.gloas_fork_epoch && epoch < target { + continue; + } if let Err(err) = duties.refresh_ptc(target).await { warn!( epoch = target, @@ -1787,35 +1794,39 @@ mod tests { } /// The committee schedule is fetched for gloas epochs only, so a chain that - /// has not forked is never asked, and the lookahead is gated on its own - /// epoch: the last pre-gloas epoch looks ahead into the first gloas one. + /// has not forked is never asked. The fork epoch's is not fetched as a + /// lookahead from the last pre-gloas epoch, since a node cannot know it + /// before the fork, but at the fork epoch itself, with the next as the + /// lookahead. #[tokio::test] async fn the_committee_schedule_is_fetched_for_gloas_epochs_only() { let node = Arc::new( node() + .with_duties(5, Root::repeat_byte(2), Vec::new()) + .with_proposers(4, Root::repeat_byte(2), Vec::new()) .with_ptc_duties(4, Root::repeat_byte(1), vec![ptc_duty(11, 130)]) .with_ptc_duties(5, Root::repeat_byte(1), Vec::new()), ); let mut duties = DutiesService::new(node.clone(), Vec::new()); let keys = [pubkey(1), pubkey(2)]; + let gloas_at_4 = gloas_context(4); - // Epoch 3 is the last before gloas at epoch 4: only the lookahead asks. - node.set_duties(3, Root::repeat_byte(1), Vec::new()); - refresh_epoch( - &node, - &mut duties, - &keys, - 3, - &settings(None), - &gloas_context(4), - ) - .await - .expect("refreshes"); + // Epoch 3 is the last before gloas at epoch 4: nothing is asked. + refresh_epoch(&node, &mut duties, &keys, 3, &settings(None), &gloas_at_4) + .await + .expect("refreshes"); assert_eq!( - node.ptc_call_count(), - 1, - "only epoch 4, the first gloas one" + node.ptc_requested_epochs(), + Vec::::new(), + "the fork epoch's committee is not known before the fork" ); + assert!(duties.ptc_at_slot(130, 4).is_empty()); + + // At the fork epoch, both it and the next are asked. + refresh_epoch(&node, &mut duties, &keys, 4, &settings(None), &gloas_at_4) + .await + .expect("refreshes"); + assert_eq!(node.ptc_requested_epochs(), vec![4, 5]); assert_eq!(duties.ptc_at_slot(130, 4).len(), 1); // A chain with no gloas scheduled is never asked at all. diff --git a/docs/cli.md b/docs/cli.md index 320dd1ad0..853a7a5d7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -421,7 +421,10 @@ the envelope is logged at error and counted, and does not fail the proposal. A block that commits to a builder's bid is left alone. Each epoch it also fetches the payload timeliness committee duties for the -validators it holds, for this epoch and the next when they are gloas ones. A member sleeps until `PAYLOAD_ATTESTATION_DUE_BPS` of the +validators it holds, for this epoch and the next when they are gloas ones. The +gloas fork epoch's are the exception: a node cannot know them before the fork, +so they are fetched at the fork epoch's first slot rather than as the previous +epoch's lookahead. A member sleeps until `PAYLOAD_ATTESTATION_DUE_BPS` of the slot, asks the node for the payload attestation data, signs it under the PTC domain and submits it. A `204` (the node knows no block for the slot) means no vote. So a gloas slot runs in diff --git a/docs/rpc.md b/docs/rpc.md index c7e9012dc..b68039d40 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -845,7 +845,15 @@ Fulu (`blocks/{slot}`) and gloas (`produceBlockV4`) do the same, and the under a fulu head, is answered from a copy of the head state advanced to the epoch's first slot, crossing the gloas upgrade where it falls, on a blocking thread. A gloas epoch more than one before the head state's is a `400`. - `dependent_root` is defined as for attester duties. There is no `503`. + The gloas fork epoch is a `400` until its first slot has started by the wall + clock, allowing `MAXIMUM_GOSSIP_CLOCK_DISPARITY` (500 ms), while the head is + still before it: its committee is computed at the fork from the effective + balances the last pre-gloas epoch ends with, so it can change until then + (Lighthouse refuses it the same way). `dependent_root` is defined as for + attester duties, except for the fork epoch, whose root is the block root at + the last slot before the fork (the genesis block's when gloas starts at + genesis). + There is no `503`. - **`GET /eth/v1/validator/payload_attestation_data?slot=`** answers `{version: "gloas", data: {beacon_block_root, slot, payload_present, blob_data_available}}` as JSON, or the SSZ `PayloadAttestationData` on From e39a0fb81e6c8c65fe75ac273faf9d66abcbbcde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:16:05 -0300 Subject: [PATCH 2/2] fix(validator): fetch a missing PTC schedule again after each slot's duties The gloas fork epoch's payload timeliness committee is now fetched once, at the fork epoch's first slot, since a node cannot know it before the fork. refresh_ptc logs a failed fetch and moves on, and nothing retried it within the epoch, so one failed request there left the client with no PTC duties for the whole first gloas epoch. Every other epoch has two chances, the lookahead and the current fetch. After each slot's duties, the duty loop now fetches the current epoch's schedule again when it is a gloas epoch and none is held, bounded by the end of the slot so it never delays a duty. It runs only once the epoch's refresh has succeeded, since a failed refresh is retried every slot anyway. A schedule that came back empty counts as held. Failed retries log at debug, as refresh_ptc has already warned. --- crates/validator/src/duties.rs | 6 +++ crates/validator/src/lib.rs | 93 +++++++++++++++++++++++++++++++++- docs/cli.md | 3 +- 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/crates/validator/src/duties.rs b/crates/validator/src/duties.rs index cd8b3372b..6bdf8b138 100644 --- a/crates/validator/src/duties.rs +++ b/crates/validator/src/duties.rs @@ -314,6 +314,12 @@ impl DutiesService { self.sync.retain(|held, _| *held >= period); } + /// Whether a payload timeliness committee schedule is held for `epoch`, + /// an empty one included: a fetch that found no duties still answered. + pub fn holds_ptc(&self, epoch: Epoch) -> bool { + self.ptc.contains_key(&epoch) + } + /// Forget payload timeliness committee duties for epochs before `epoch`. pub fn prune_ptc_before(&mut self, epoch: Epoch) { self.ptc.retain(|held, _| *held >= epoch); diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index f579ad8a3..a57624d2e 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -552,6 +552,19 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { // them. Off the slot loop like the sends above. The refresh's slot // too, although its refresh sent them: see `Preparations`. preparations.after_slot(slot, &resolved, &store); + + // This epoch's payload committee schedule again, while none is + // held. The gloas fork epoch's is fetched at its first slot only + // (see `refresh_ptc`), so without this one failed request there + // would cost the whole first gloas epoch's votes; for any other + // epoch it covers a lookahead and a refresh that both failed. + // After the slot's duties and bounded by the slot's end, so it + // never delays a duty. Only once this epoch's refresh got through: + // until then the refresh itself runs again every slot. + if last_epoch == Some(epoch) { + let budget = clock.remaining_in(slot, SystemTime::now()); + retry_missing_ptc(&mut duties, epoch, budget, &context).await; + } } }; @@ -1253,7 +1266,9 @@ fn is_registrable(status: &str) -> bool { /// answers `400` before). It is fetched at the fork epoch's own first slot, /// with the epoch after it as the lookahead. A failed fetch is logged and /// shrugged off, like the proposer and lookahead ones: it must not cost this -/// epoch's attester schedule, which is already in hand. +/// epoch's attester schedule, which is already in hand. The current epoch's is +/// then fetched again after each slot's duties until one is held +/// ([`retry_missing_ptc`]). async fn refresh_ptc( duties: &mut DutiesService, epoch: u64, @@ -1277,6 +1292,37 @@ async fn refresh_ptc( duties.prune_ptc_before(epoch); } +/// Fetch this epoch's payload timeliness committee schedule again, within +/// `budget`, when it is a gloas epoch and none is held. +/// +/// The duty loop calls it after every slot's duties, so a failed fetch costs +/// the votes of the slots until the next one succeeds rather than the rest of +/// the epoch. A failure here is logged at debug: `refresh_ptc` already warned +/// once, and this runs again every slot until it succeeds. Dropping the fetch +/// at the deadline is safe, since `DutiesService::refresh_ptc` changes nothing +/// until its request has answered. +async fn retry_missing_ptc( + duties: &mut DutiesService, + epoch: u64, + budget: std::time::Duration, + context: &SigningContext, +) { + if context.config.fork_at_epoch(epoch) < ethlambda_types::beacon::fork::ForkName::Gloas + || duties.holds_ptc(epoch) + { + return; + } + match tokio::time::timeout(budget, duties.refresh_ptc(epoch)).await { + Ok(Ok(())) => {} + Ok(Err(err)) => { + debug!(%epoch, %err, "Payload committee duties retry failed; trying again next slot"); + } + Err(_) => { + debug!(%epoch, "Payload committee duties retry ran out of slot; trying again next slot"); + } + } +} + /// How many epochs before a period boundary the next period's subnets are /// joined: lighthouse's lookahead, at the specification's upper bound /// `SYNC_COMMITTEE_SUBNET_COUNT` for the random early-join window. @@ -1842,6 +1888,51 @@ mod tests { node() } + /// The fork epoch's committee is fetched once, at its first slot, so a + /// fetch that failed there is retried after each slot until a schedule is + /// held, and then no more. A pre-gloas epoch is never asked. + #[tokio::test] + async fn a_missing_committee_schedule_is_fetched_again() { + // No duties stored for epoch 4 yet, so the mock refuses the request. + let node = Arc::new(node()); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + duties.set_indices(vec![11]); + let gloas_at_4 = gloas_context(4); + let budget = std::time::Duration::from_secs(1); + + retry_missing_ptc(&mut duties, 4, budget, &gloas_at_4).await; + assert_eq!(node.ptc_requested_epochs(), vec![4]); + assert!(!duties.holds_ptc(4), "a failed retry holds nothing"); + + // The node answers now: the next slot's retry gets the schedule. + node.set_ptc_duties(4, Root::repeat_byte(1), vec![ptc_duty(11, 130)]); + retry_missing_ptc(&mut duties, 4, budget, &gloas_at_4).await; + assert_eq!(node.ptc_requested_epochs(), vec![4, 4]); + assert_eq!(duties.ptc_at_slot(130, 4).len(), 1); + + // Held: no further request. + retry_missing_ptc(&mut duties, 4, budget, &gloas_at_4).await; + assert_eq!(node.ptc_requested_epochs(), vec![4, 4]); + + // Before gloas there is no committee to ask about. + retry_missing_ptc(&mut duties, 3, budget, &gloas_at_4).await; + assert_eq!(node.ptc_requested_epochs(), vec![4, 4]); + } + + /// A schedule that came back empty is still a schedule: nothing to retry. + #[tokio::test] + async fn an_empty_committee_schedule_is_not_fetched_again() { + let node = Arc::new(node().with_ptc_duties(4, Root::repeat_byte(1), Vec::new())); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + duties.set_indices(vec![11]); + let gloas_at_4 = gloas_context(4); + let budget = std::time::Duration::from_secs(1); + + retry_missing_ptc(&mut duties, 4, budget, &gloas_at_4).await; + retry_missing_ptc(&mut duties, 4, budget, &gloas_at_4).await; + assert_eq!(node.ptc_requested_epochs(), vec![4]); + } + /// A failed committee fetch must not cost the epoch's attester schedule. #[tokio::test] async fn a_failed_committee_fetch_does_not_fail_the_refresh() { diff --git a/docs/cli.md b/docs/cli.md index 853a7a5d7..0ff35f0e7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -424,7 +424,8 @@ Each epoch it also fetches the payload timeliness committee duties for the validators it holds, for this epoch and the next when they are gloas ones. The gloas fork epoch's are the exception: a node cannot know them before the fork, so they are fetched at the fork epoch's first slot rather than as the previous -epoch's lookahead. A member sleeps until `PAYLOAD_ATTESTATION_DUE_BPS` of the +epoch's lookahead. When an epoch's schedule could not be fetched, the client +asks again after each slot's duties until it has one. A member sleeps until `PAYLOAD_ATTESTATION_DUE_BPS` of the slot, asks the node for the payload attestation data, signs it under the PTC domain and submits it. A `204` (the node knows no block for the slot) means no vote. So a gloas slot runs in