From 039fc179879d873abd90cc5de4fa4afb8d71d79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:28:17 -0300 Subject: [PATCH 1/7] fix(p2p): keep a column lookup open until every column has arrived A DataColumnsByRoot lookup is split across the peers that custody its columns, with the columns no known custodian covers going to one random peer, but it was retired on the first non-empty answer. Every column asked of another peer in the same round then waited for the chain actor's next per-slot re-drive, ~12 s on mainnet timing, however soon it could have come. An answer now takes its columns off the lookup, and the lookup is retired only once nothing is left. A short answer ends its request the way a failure does, so the rest is asked again on the existing backoff ladder; the peer is charged only if it left out a column it was itself asked for. Found on a kurtosis devnet (gloas at epoch 2): one ethlambda node had a known custodian for 2 of its 8 custody columns; about one envelope in twenty waited ~11.7 s for its columns, which made its PTC vote payload_present=false for payloads the network had on time and orphaned its next proposal, built on the EMPTY parent. --- crates/net/p2p/src/lib.rs | 8 +- crates/net/p2p/src/req_resp/handlers.rs | 169 +++++++++++++++++++++++- 2 files changed, 167 insertions(+), 10 deletions(-) diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index cc9ca2bd1..bed960ae7 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -387,9 +387,11 @@ pub(crate) enum PendingRequestKind { /// A `DataColumnsByRoot` lookup for this block's missing columns. Carries /// the root and the columns this one request asked its peer for: the /// attempts and failed-peer set live in `pending_column_requests`, keyed - /// by the root, but one lookup fans out across peers with a subset each and - /// retires its entry on the first answer, so what a later answer may - /// contain has to travel with the request itself. + /// by the root, but one lookup fans out across peers with a subset each, + /// and an answer is judged against its own request's subset (whether its + /// peer left anything out), and may arrive after a sibling's answer has + /// already retired the entry, so what it may contain has to travel with + /// the request itself. Columns(H256, Vec), /// A `DataColumnsByRange` sweep for a range sync batch, covering the same /// slots the matching `BlocksByRange` asked for. diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 5801a84f2..d47b131be 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -1577,13 +1577,26 @@ async fn handle_column_fetch_failure( return; }; - match retire_column_attempt(pending, peer) { + let outcome = retire_column_attempt(pending, peer); + follow_column_attempt(server, block_root, peer, outcome, ctx); +} + +/// Carry out what [`retire_column_attempt`] or [`settle_column_answer`] +/// decided for `block_root`'s lookup. +fn follow_column_attempt( + server: &mut P2PServer, + block_root: H256, + peer: PeerId, + outcome: ColumnAttemptOutcome, + ctx: &Context, +) { + match outcome { ColumnAttemptOutcome::RoundIncomplete { in_flight } => { trace!( %block_root, %peer, in_flight, - "One peer of this column lookup failed; waiting for the rest of the round" + "One peer of this column lookup reported back; waiting for the rest of the round" ); } ColumnAttemptOutcome::GiveUp { attempts } => { @@ -1593,7 +1606,7 @@ async fn handle_column_fetch_failure( metrics::inc_data_column_fetch_failure("max_retries"); } ColumnAttemptOutcome::Retry { attempts, backoff } => { - debug!(%block_root, %peer, attempts, ?backoff, "Data column fetch failed, scheduling retry"); + debug!(%block_root, %peer, attempts, ?backoff, "Data column lookup still missing columns, scheduling retry"); send_after( backoff, ctx.clone(), @@ -1629,6 +1642,43 @@ enum ColumnAttemptOutcome { /// fan-out per failure, which squares with every retry. fn retire_column_attempt(pending: &mut PendingColumnRequest, peer: PeerId) -> ColumnAttemptOutcome { pending.failed_peers.insert(peer); + close_column_request(pending) +} + +/// Take a non-empty answer's columns off what `pending` still asks for, and +/// say what follows: `None` once nothing is left, for the caller to retire the +/// lookup. +/// +/// A lookup is split across the peers that custody its columns, so one peer's +/// answer is often only part of it. Retiring the lookup on that first answer +/// left every column asked of another peer in the same round to wait for the +/// chain actor's next per-slot re-drive, however soon it could have come: on a +/// devnet, about one gloas envelope in twenty waited ~11.7 s for its columns. +/// A short answer now ends its request the way a failure does, so the rest is +/// asked again on the same ladder. +/// +/// Charged to `peer` only if it left out a column it was itself asked for. A +/// peer that sent its whole share did nothing wrong: what remains is another +/// request's, or a column merged into the lookup after this request went out. +fn settle_column_answer( + pending: &mut PendingColumnRequest, + peer: PeerId, + requested: &[u64], + delivered: &[u64], +) -> Option { + pending.columns.retain(|column| !delivered.contains(column)); + if pending.columns.is_empty() { + return None; + } + if requested.iter().any(|column| !delivered.contains(column)) { + pending.failed_peers.insert(peer); + } + Some(close_column_request(pending)) +} + +/// One request of `pending`'s round has reported back without completing the +/// lookup: the round's outcome once it was the last one open. +fn close_column_request(pending: &mut PendingColumnRequest) -> ColumnAttemptOutcome { pending.in_flight = pending.in_flight.saturating_sub(1); if pending.in_flight > 0 { @@ -2550,10 +2600,22 @@ async fn handle_data_column_sidecars_response( return; } - // The lookup is done, whether or not every requested column arrived; see - // this function's own doc comment for why a partial answer still retires - // the entry rather than triggering a same-lookup retry for the rest. - server.pending_column_requests.remove(&block_root); + // The lookup is done once every column it asks for has arrived; a short + // answer leaves it open for the rest (see `settle_column_answer`). No entry + // means a sibling's answer already completed it, or the ladder gave up: + // the sidecars are still forwarded below. + let delivered: Vec = sidecars.iter().map(|sidecar| sidecar.index()).collect(); + let settled = server + .pending_column_requests + .get_mut(&block_root) + .map(|pending| settle_column_answer(pending, peer, requested, &delivered)); + match settled { + Some(None) => { + server.pending_column_requests.remove(&block_root); + } + Some(Some(outcome)) => follow_column_attempt(server, block_root, peer, outcome, ctx), + None => {} + } if server.blockchain.is_none() { debug!(%peer, %block_root, "No blockchain handler available"); @@ -3079,6 +3141,99 @@ pub(crate) mod tests { ); } + fn column_lookup(columns: Vec, in_flight: usize) -> PendingColumnRequest { + PendingColumnRequest { + columns, + attempts: 1, + failed_peers: HashSet::new(), + in_flight, + last_asked: Instant::now(), + } + } + + #[test] + fn an_answer_carrying_every_missing_column_completes_the_lookup() { + let mut pending = column_lookup(vec![1, 2], 1); + let peer = PeerId::random(); + + assert_eq!( + settle_column_answer(&mut pending, peer, &[1, 2], &[1, 2]), + None + ); + assert!(pending.failed_peers.is_empty()); + } + + #[test] + fn a_short_answer_keeps_the_lookup_and_retries_the_rest() { + // The lookup used to be retired by any non-empty answer, leaving the + // columns this peer left out for the chain actor's next per-slot + // re-drive. + let mut pending = column_lookup(vec![1, 2, 3], 1); + let peer = PeerId::random(); + + assert_eq!( + settle_column_answer(&mut pending, peer, &[1, 2, 3], &[1]), + Some(ColumnAttemptOutcome::Retry { + attempts: 1, + backoff: Duration::from_millis(INITIAL_BACKOFF_MS), + }) + ); + assert_eq!(pending.columns, vec![2, 3], "only what is still missing"); + assert!( + pending.failed_peers.contains(&peer), + "it left out a column it was asked for" + ); + } + + #[test] + fn a_split_lookup_waits_for_every_share_and_charges_only_the_short_one() { + // Custody-aware splitting: a custodian of column 1 and a random peer + // for 2 and 3, the shape a node gets when no peer custodies most of + // its columns. The custodian answering first must not retire the + // lookup the other request still serves. + let mut pending = column_lookup(vec![1, 2, 3], 2); + let custodian = PeerId::random(); + let other = PeerId::random(); + + assert_eq!( + settle_column_answer(&mut pending, custodian, &[1], &[1]), + Some(ColumnAttemptOutcome::RoundIncomplete { in_flight: 1 }) + ); + assert!( + pending.failed_peers.is_empty(), + "the custodian sent its whole share" + ); + + assert_eq!( + settle_column_answer(&mut pending, other, &[2, 3], &[2]), + Some(ColumnAttemptOutcome::Retry { + attempts: 1, + backoff: Duration::from_millis(INITIAL_BACKOFF_MS), + }) + ); + assert_eq!(pending.columns, vec![3]); + assert_eq!(pending.failed_peers, HashSet::from([other])); + } + + #[test] + fn a_whole_share_still_retries_a_column_merged_in_after_the_request() { + // `fetch_missing_columns` merges a re-ask's new column into the lookup + // in flight; the request on the wire cannot carry it, so the round's + // end has to ask for it. + let mut pending = column_lookup(vec![1, 4], 1); + let peer = PeerId::random(); + + assert_eq!( + settle_column_answer(&mut pending, peer, &[1], &[1]), + Some(ColumnAttemptOutcome::Retry { + attempts: 1, + backoff: Duration::from_millis(INITIAL_BACKOFF_MS), + }) + ); + assert_eq!(pending.columns, vec![4]); + assert!(pending.failed_peers.is_empty()); + } + #[test] fn an_out_of_range_custody_group_count_records_nothing() { // A count outside CUSTODY_REQUIREMENT..=NUMBER_OF_CUSTODY_GROUPS From ce1dc39e168d6a661692380a90f2aa5de0796707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:50:35 -0300 Subject: [PATCH 2/7] fix(p2p): settle each column request on its own columns and on what passed the checks The lookup counted its open requests (`in_flight`) and acted only once the last one had reported. A short answer's leftover waited out a sibling's request timeout, productive rounds were charged as failures, a custodian that answered short at the tip was excluded for the rest of the lookup, and a replaced lookup's late report closed one of the new lookup's requests. - Each lookup records which column is on the wire with which peer (`asked`), so a request settles only its own columns and whatever it left out is asked again at once. A retry asks only what nothing open covers, so sibling failures still cannot multiply into a fan-out each; they coalesce into the one retry already due (`retry_due`). - Only a request that left out a column it was asked for spends an attempt and doubles the backoff. A whole share spends nothing, and a column merged in afterwards is asked at once, from the merge itself too. - A peer that answers without a column is not written off. It is recorded in `missed`. While the block is at the tip (it may not have the column yet) a retry asks every other candidate for that column first, then comes back to it; past the tip it is not asked for that column again. A failed request or a refused sidecar still writes the peer off. - Columns no known custodian covers go to a peer of unknown custody, as the range path already does, rather than to a uniformly random one. Only when none is connected are they asked of a peer of known custody, once per lookup. - A lookup with nobody left to ask, or whose ladder ran out on columns no connected peer is known to custody, ends at debug level and counts as `no_custodian` on `lean_data_column_fetch_failures_total`, not as an error and `max_retries`. - An answer is settled after the chain checks, on what passed them (`column_checks::check_and_forward_then` reports each sidecar's verdict back to the actor), not on unverified sidecars. - Before asking again, the columns the store already holds are dropped. - Requests and retry timers carry their lookup's id, so a replaced lookup's report or timer settles nothing of its successor. - `ColumnAttemptOutcome::Complete` replaces `Option>`, and `retire_column_lookup` is the one place a lookup ends. - Docs: the by-root response doc, `pending_column_requests`, `PendingRequestKind::Columns`, `STALE_COLUMN_LOOKUP`, the outcome enum, and the new metric reason in docs/metrics.md. - Tests for the arithmetic, and for the wiring on a beacon test server: completion after the checks, the no-entry path still forwarding, giving up from an answer, the store check, the merge in `fetch_missing_columns` and the stale replacement, the stale report and timer, a short peer asked again at the tip and once past it, and the once-per-lookup ask beyond custody. --- crates/net/p2p/src/beacon/column_checks.rs | 153 +- crates/net/p2p/src/lib.rs | 214 ++- crates/net/p2p/src/metrics.rs | 7 +- crates/net/p2p/src/req_resp/handlers.rs | 1647 +++++++++++++++----- docs/metrics.md | 11 +- 5 files changed, 1602 insertions(+), 430 deletions(-) diff --git a/crates/net/p2p/src/beacon/column_checks.rs b/crates/net/p2p/src/beacon/column_checks.rs index 91fbd1efe..43cabd912 100644 --- a/crates/net/p2p/src/beacon/column_checks.rs +++ b/crates/net/p2p/src/beacon/column_checks.rs @@ -26,6 +26,38 @@ use tracing::{error, warn}; use crate::{P2PServer, metrics}; +/// What the checks made of one fetched sidecar, as the lookup that asked for +/// it needs to know: whether its column is now here, and whether the peer that +/// sent it is to blame if not. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ColumnVerdict { + /// Passed and went to the chain actor, or was already stored: the column + /// is here. + Delivered, + /// Not judged, through no fault of the sender: parked until its block can + /// judge it, dropped for a reason that is not the sender's (an `Ignore`), + /// or never checked, because this node had no chain actor to check it for + /// or its check panicked. + Undecided, + /// Refused as invalid: the sender sent something it should not have. + Rejected, +} + +impl From<&ChainVerdict> for ColumnVerdict { + fn from(verdict: &ChainVerdict) -> Self { + match verdict { + ChainVerdict::Keep + | ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::AlreadyStored)) => Self::Delivered, + ChainVerdict::Drop(Outcome::Reject(_)) => Self::Rejected, + ChainVerdict::AwaitParent | ChainVerdict::Drop(_) => Self::Undecided, + } + } +} + +/// A completion callback: each sidecar's column index with its +/// [`ColumnVerdict`]. +pub(crate) type Done = Box) + Send>; + /// Run the chain checks on `sidecars`, then send the chain actor the ones that /// passed as one batch and the ones waiting on a block as another. /// @@ -33,33 +65,62 @@ use crate::{P2PServer, metrics}; /// goes on handling swarm events while a range batch's KZG proofs are /// verified. pub(crate) fn check_and_forward(server: &P2PServer, sidecars: Vec) { - if sidecars.is_empty() { - return; - } - let Some(blockchain) = server.blockchain.clone() else { - return; + check_and_forward_then(server, sidecars, None); +} + +/// [`check_and_forward`], then `done` with every sidecar's verdict, once the +/// ones that passed have been handed to the chain actor. +/// +/// This is how a `DataColumnsByRoot` lookup settles on what passed rather than +/// on what a peer sent: a sidecar with the right root and index can still +/// carry bad cells or proofs, and a gloas one carries no signature at all. +/// `done` runs on every path, with a sidecar that was never judged reported as +/// [`ColumnVerdict::Undecided`]: a lookup left waiting on a verdict that never +/// comes would keep its columns covered until the lookup goes stale. +pub(crate) fn check_and_forward_then( + server: &P2PServer, + sidecars: Vec, + done: Option, +) { + let run_done = |done: Option, verdicts: Vec<(u64, ColumnVerdict)>| { + if let Some(done) = done { + done(verdicts); + } + }; + let blockchain = match server.blockchain.clone() { + Some(blockchain) if !sidecars.is_empty() => blockchain, + _ => { + run_done(done, undecided(&sidecars)); + return; + } }; let store = server.store.clone(); let permits = server.column_check_permits.clone(); tokio::spawn(async move { let mut checks = Vec::with_capacity(sidecars.len()); - for sidecar in sidecars { + let mut verdicts = Vec::with_capacity(sidecars.len()); + let mut sidecars = sidecars.into_iter(); + while let Some(sidecar) = sidecars.next() { // The semaphore is never closed, so this only returns once a // permit is free. let Ok(permit) = permits.clone().acquire_owned().await else { - return; + verdicts.push((sidecar.index(), ColumnVerdict::Undecided)); + verdicts.extend(undecided(sidecars.as_slice())); + break; }; let store = store.clone(); - checks.push(tokio::task::spawn_blocking(move || { + let index = sidecar.index(); + let check = tokio::task::spawn_blocking(move || { let _permit = permit; let verdict = column::chain_checks_for(&store, &sidecar, unix_now_ms()); (sidecar, verdict) - })); + }); + checks.push((index, check)); } let mut keep = Vec::new(); let mut awaiting_parent = Vec::new(); - for check in checks { + for (index, check) in checks { // A panic inside the checks (a `Store` read's `.expect()` on a DB // error, say) costs that one sidecar, which is what dropping it // would have done anyway. Tokio has already caught it. @@ -67,9 +128,11 @@ pub(crate) fn check_and_forward(server: &P2PServer, sidecars: Vec checked, Err(err) => { error!(%err, "A data column check panicked; dropping the sidecar"); + verdicts.push((index, ColumnVerdict::Undecided)); continue; } }; + verdicts.push((index, ColumnVerdict::from(&verdict))); match verdict { ChainVerdict::Keep => keep.push(sidecar), ChainVerdict::AwaitParent => awaiting_parent.push(sidecar), @@ -89,9 +152,18 @@ pub(crate) fn check_and_forward(server: &P2PServer, sidecars: Vec Vec<(u64, ColumnVerdict)> { + sidecars + .iter() + .map(|sidecar| (sidecar.index(), ColumnVerdict::Undecided)) + .collect() +} + /// Count a dropped sidecar under its reason. /// /// An already stored one is not counted: consecutive range batches ask for @@ -106,7 +178,7 @@ fn count_drop(outcome: Outcome) { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::sync::Arc; use ethlambda_network_api::{ @@ -124,13 +196,13 @@ mod tests { /// What [`check_and_forward`] sent the chain actor. #[derive(Debug, PartialEq)] - enum Forwarded { + pub(crate) enum Forwarded { Checked(Vec), AwaitingParent(Vec), } /// A chain actor stand-in that reports every sidecar batch it receives. - struct RecordingChain(mpsc::UnboundedSender); + pub(crate) struct RecordingChain(pub(crate) mpsc::UnboundedSender); impl P2PToBlockChain for RecordingChain { fn new_block( @@ -257,4 +329,59 @@ mod tests { drop(server); assert_eq!(received.recv().await, None); } + + /// The completion callback hears every sidecar's verdict: one already + /// stored is here, a malformed one is its sender's fault, and one waiting + /// on its parent is neither. + #[tokio::test] + async fn the_callback_reports_what_the_checks_made_of_each_sidecar() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let (sender, _received) = mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(RecordingChain(sender))); + + let stored = DataColumnSidecar::Fulu(valid_shaped_sidecar(5, 0)); + server.store.put_data_column(&stored).expect("stored"); + let mut malformed = valid_shaped_sidecar(5, 1); + malformed.kzg_commitments = Default::default(); + let orphan = valid_shaped_sidecar(5, 2); + + let (verdicts, mut reported) = mpsc::unbounded_channel(); + let done: Done = Box::new(move |checked| { + let _ = verdicts.send(checked); + }); + let sidecars = vec![ + stored, + DataColumnSidecar::Fulu(malformed), + DataColumnSidecar::Fulu(orphan), + ]; + check_and_forward_then(&server, sidecars, Some(done)); + + assert_eq!( + reported.recv().await, + Some(vec![ + (0, ColumnVerdict::Delivered), + (1, ColumnVerdict::Rejected), + (2, ColumnVerdict::Undecided), + ]) + ); + } + + /// With no chain actor to check for, the callback still runs, so a lookup + /// waiting on it is not left with its columns on the wire. + #[tokio::test] + async fn the_callback_runs_when_nothing_can_be_checked() { + let server = unconnected_beacon_server(Config::mainnet(), 0).await; + let (verdicts, mut reported) = mpsc::unbounded_channel(); + let done: Done = Box::new(move |checked| { + let _ = verdicts.send(checked); + }); + + let sidecar = DataColumnSidecar::Fulu(valid_shaped_sidecar(5, 3)); + check_and_forward_then(&server, vec![sidecar], Some(done)); + + assert_eq!( + reported.recv().await, + Some(vec![(3, ColumnVerdict::Undecided)]) + ); + } } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index bed960ae7..d4d794b91 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -105,7 +105,7 @@ use crate::{ req_resp::{ Codec, MAX_COMPRESSED_PAYLOAD_SIZE, ReqResp, ReqRespEvent, Request, build_status, fetch_block_from_peer, fetch_data_columns_from_peer, fetch_envelope_from_peer, - handlers::{columns_custodied_by, resume_range_batch_held_for_custody}, + handlers::{ColumnReport, columns_custodied_by, resume_range_batch_held_for_custody}, }, swarm_adapter::SwarmHandle, }; @@ -166,6 +166,12 @@ const BACKOFF_MULTIPLIER: u64 = 2; /// backoffs sum to a few seconds, plus each attempt's round trips) and shorter /// than a mainnet slot, so a re-drive arriving on the next tick finds either a /// live lookup or no entry at all. +/// +/// Shorter than the request-response timeout (libp2p's default 10 s), so a +/// replaced lookup's requests can still report back after the new one has +/// started. Each request carries its lookup's id +/// ([`PendingRequestKind::Columns`]), so such a report delivers its columns +/// but settles nothing of the new lookup's. const STALE_COLUMN_LOOKUP: Duration = Duration::from_secs(8); /// How many peers of unknown custody one `DataColumnsByRange` prefetch may @@ -356,26 +362,80 @@ pub(crate) struct PendingRequest { /// A block-root fetch's counterpart for a column lookup: the same /// attempt/failed-peer bookkeeping [`PendingRequest`] carries, plus what -/// [`PendingRequest`] never needed to. `columns` is the request body a retry -/// resends, since (unlike a block-root fetch, whose body is the root already -/// keying this map) a column request also names which columns. +/// [`PendingRequest`] never needed to, since (unlike a block-root fetch, whose +/// body is the root already keying this map) a column request also names +/// which columns, and one lookup is split across several peers. pub(crate) struct PendingColumnRequest { + /// The columns still missing: asked and not yet answered, or waiting to be + /// asked. A column leaves once a sidecar for it has passed the checks, or + /// once the store has it when a retry is about to ask. pub(crate) columns: Vec, + /// This lookup's own id, which each of its requests carries + /// ([`PendingRequestKind::Columns`]). A request from a lookup this one + /// replaced (see [`STALE_COLUMN_LOOKUP`]) can still report back, and must + /// not settle this one's requests or spend its attempts. + pub(crate) lookup: u64, + /// Rounds spent on a request that left out a column it was asked for. A + /// request that delivered everything it asked for spends nothing. pub(crate) attempts: u32, + /// Peers not asked again in this lookup: a request to them failed, or + /// they sent a sidecar the checks refused. A peer that answered without a + /// column is not here: at the tip it may simply not have it yet. pub(crate) failed_peers: HashSet, - /// Requests sent for this root and not yet answered or failed. + /// `(peer, column)` pairs asked in this lookup that did not deliver. While + /// the block is at the tip a retry prefers another peer for that column + /// and comes back to this one only when there is none; past the tip it + /// does not come back (see `plan_column_requests`). + pub(crate) missed: HashSet<(PeerId, u64)>, + /// Whether this lookup has asked a peer for columns outside its known + /// custody, which it does once at most (see `plan_column_requests`). + pub(crate) asked_beyond_custody: bool, + /// The columns on the wire, each with the peer asked for it. /// - /// One attempt now fans out across the peers that custody the columns, so - /// a single lookup can have several requests open at once. Without this - /// count each of their failures would schedule its own retry, and each - /// retry would fan out again: one unanswered lookup against eight - /// custodians becomes eight retries, then sixty-four. A retry is scheduled - /// only when the last outstanding request of the round reports back, so an - /// attempt still costs exactly one retry however wide it was. - pub(crate) in_flight: usize, + /// One lookup fans out across the peers that custody its columns, so it + /// can have several requests open at once, and they ask disjoint sets: a + /// column is asked again only once the request that asked for it has + /// reported back. That is what lets a request's leftover be asked at once + /// without waiting for its siblings, and why their failures cannot + /// multiply into one fan-out each: a retry asks only what is not on the + /// wire. A request is "on the wire" until its answer has been through the + /// checks, so a column being verified is not asked again meanwhile. + pub(crate) asked: HashMap, + /// A retry is scheduled. Whatever another request leaves out meanwhile + /// rides on it rather than spending an attempt of its own. + pub(crate) retry_due: bool, /// When this lookup last put requests on the wire, for /// [`STALE_COLUMN_LOOKUP`] to measure against. pub(crate) last_asked: Instant, + /// The block's slot, the other half of the store's column key, read once + /// when a retry first checks the store. + pub(crate) slot: Option, +} + +impl PendingColumnRequest { + pub(crate) fn new(columns: Vec, lookup: u64) -> Self { + Self { + columns, + lookup, + attempts: 1, + failed_peers: HashSet::new(), + missed: HashSet::new(), + asked_beyond_custody: false, + asked: HashMap::new(), + retry_due: false, + last_asked: Instant::now(), + slot: None, + } + } + + /// The columns still missing that no open request asks for. + pub(crate) fn unasked(&self) -> Vec { + self.columns + .iter() + .copied() + .filter(|column| !self.asked.contains_key(column)) + .collect() + } } pub(crate) enum PendingRequestKind { @@ -384,15 +444,19 @@ pub(crate) enum PendingRequestKind { start_slot: u64, end_slot: u64, }, - /// A `DataColumnsByRoot` lookup for this block's missing columns. Carries - /// the root and the columns this one request asked its peer for: the - /// attempts and failed-peer set live in `pending_column_requests`, keyed - /// by the root, but one lookup fans out across peers with a subset each, - /// and an answer is judged against its own request's subset (whether its - /// peer left anything out), and may arrive after a sibling's answer has - /// already retired the entry, so what it may contain has to travel with - /// the request itself. - Columns(H256, Vec), + /// A `DataColumnsByRoot` request of one lookup for this block's missing + /// columns. The lookup's bookkeeping lives in `pending_column_requests`, + /// keyed by the root, but one lookup fans out across peers with a subset + /// each, and an answer is judged against its own request's subset + /// (whether its peer left anything out), so the subset travels with the + /// request. So does the lookup's id: a lookup can be replaced while its + /// requests are still open, and their reports must not be taken for the + /// new one's. + Columns { + block_root: H256, + columns: Vec, + lookup: u64, + }, /// A `DataColumnsByRange` sweep for a range sync batch, covering the same /// slots the matching `BlocksByRange` asked for. /// @@ -1377,7 +1441,9 @@ pub struct P2PServer { /// One entry per block root with an in-flight or backed-off /// `DataColumnsByRoot` lookup. Mirrors `pending_root_requests`'s role for /// the block path: `fetch_missing_columns` dedupes against it, and - /// `handle_column_fetch_failure` is the only place an entry is retired. + /// `retire_column_lookup` is the only place an entry is retired, once its + /// columns have all arrived, its ladder has run out, or nobody is left to + /// ask (`fetch_missing_columns` also drops a stale one to replace it). pub(crate) pending_column_requests: HashMap, pub(crate) outbound_requests: HashMap, pub(crate) range_sync_state: Option, @@ -1544,7 +1610,11 @@ pub(crate) trait P2PProtocol: Send + Sync { #[allow(dead_code)] // invoked via send_after, not called directly fn retry_block_fetch(&self, root: H256) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly - fn retry_data_column_fetch(&self, block_root: H256) -> Result<(), ActorError>; + fn retry_data_column_fetch(&self, block_root: H256, lookup: u64) -> Result<(), ActorError>; + /// A `DataColumnsByRoot` answer has been through the chain checks: settle + /// its lookup on what passed. + #[allow(dead_code)] // sent by the column checks' completion callback + fn column_answer_checked(&self, report: ColumnReport) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly fn retry_envelope_fetch(&self, block_root: H256) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly @@ -1623,21 +1693,16 @@ impl P2PServer { msg: p2p_protocol::RetryDataColumnFetch, _ctx: &Context, ) { - let block_root = msg.block_root; - // Same "might have completed during backoff" guard as - // `handle_retry_block_fetch`, and the same reason for it: the reissued - // request must ask for what is still missing, which - // `pending_column_requests` is the only place that remembers. - let Some(pending) = self.pending_column_requests.get(&block_root) else { - trace!(%block_root, "Data column fetch completed during backoff, skipping retry"); - return; - }; - let columns = pending.columns.clone(); + req_resp::handlers::retry_column_lookup(self, msg.block_root, msg.lookup).await; + } - if !fetch_data_columns_from_peer(self, block_root, columns).await { - tracing::error!(%block_root, "Failed to retry data column fetch, giving up"); - self.pending_column_requests.remove(&block_root); - } + #[send_handler] + async fn handle_column_answer_checked( + &mut self, + msg: p2p_protocol::ColumnAnswerChecked, + ctx: &Context, + ) { + req_resp::handlers::settle_column_request(self, msg.report, ctx); } #[send_handler] @@ -1913,14 +1978,12 @@ async fn fetch_missing_columns(server: &mut P2PServer, block_root: H256, columns // in flight with a wider column set than the first ask named, as // columns trickle in, and dropping the difference would rest on an // unenforced invariant that a re-ask is always a subset of what is - // already pending. A column added this way misses the request - // already on the wire, which cannot be widened after it was sent; - // it rides the next event for this root instead — a retry of this - // lookup, which resends whatever `columns` now holds, or, once this - // lookup resolves and the entry is gone, a fresh `FetchRequest` - // from a caller that rechecks what is still missing. + // already pending. The requests already on the wire cannot be widened + // after they were sent, so whatever none of them covers is asked for at + // once, in a request of its own, unless a retry is already due to ask + // for it. A column one of them does cover is left to that request. // - // "In flight" has to mean it, though: an entry whose round never + // "In flight" has to mean it, though: an entry whose requests never // reported back would otherwise deduplicate this root against a lookup // that will never ask anything again. Past `STALE_COLUMN_LOOKUP` the // entry is dropped and this ask starts a lookup of its own. @@ -1932,6 +1995,9 @@ async fn fetch_missing_columns(server: &mut P2PServer, block_root: H256, columns pending.columns.push(column); } } + if !pending.retry_due { + req_resp::handlers::ask_unasked_columns(server, block_root).await; + } return; } debug!(%block_root, "Replacing a data column lookup that stopped asking"); @@ -4276,4 +4342,62 @@ mod tests { "a request/response field still holds the connection the limits refused" ); } + + /// A beacon server with a custodian of column 1 and a peer that has said + /// nothing about its custody. + async fn column_server() -> (P2PServer, PeerId, PeerId) { + let mut server = test_support::unconnected_beacon_server( + ethlambda_types::beacon::config::Config::mainnet(), + 0, + ) + .await; + let holder = random_peer(); + let silent = random_peer(); + for peer in [holder, silent] { + server + .connected_peers + .insert(peer, ConnectionDirection::Inbound); + } + server.peer_custody.insert(holder, vec![1]); + (server, holder, silent) + } + + #[tokio::test] + async fn a_re_ask_that_merges_a_column_asks_for_it_at_once() { + // It used to be left for the lookup's next retry, which a hung + // request could hold back for its whole timeout. + let (mut server, holder, silent) = column_server().await; + let root = H256::repeat_byte(0x61); + + fetch_missing_columns(&mut server, root, vec![1]).await; + let lookup = server.pending_column_requests[&root].lookup; + fetch_missing_columns(&mut server, root, vec![1, 2]).await; + + let pending = &server.pending_column_requests[&root]; + assert_eq!(pending.lookup, lookup, "merged, not replaced"); + assert_eq!(pending.asked, HashMap::from([(1, holder), (2, silent)])); + assert_eq!(server.outbound_requests.len(), 2, "1 is not asked twice"); + } + + #[tokio::test] + async fn a_re_ask_replaces_a_lookup_that_stopped_asking() { + let (mut server, _, _) = column_server().await; + let root = H256::repeat_byte(0x62); + + fetch_missing_columns(&mut server, root, vec![1]).await; + let pending = server + .pending_column_requests + .get_mut(&root) + .expect("a lookup"); + let stale = pending.lookup; + pending.last_asked = Instant::now() + .checked_sub(STALE_COLUMN_LOOKUP + Duration::from_secs(1)) + .expect("the clock is past the stale window"); + fetch_missing_columns(&mut server, root, vec![1]).await; + + // A new id, so the stale lookup's requests, still open, settle nothing + // of this one's. + assert_ne!(server.pending_column_requests[&root].lookup, stale); + assert_eq!(server.outbound_requests.len(), 2); + } } diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index c8f5f5cba..0b7d2724e 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -503,8 +503,11 @@ static LEAN_DATA_COLUMN_FETCH_FAILURES_TOTAL: LazyLock = LazyLock }); /// Count one `DataColumnsByRoot` lookup this node gave up on. `reason` is -/// `"no_peers"` (nothing connected to ask) or `"max_retries"` (the retry -/// ladder ran out). +/// `"no_peers"` (nothing connected to ask), `"max_retries"` (the retry ladder +/// ran out on columns some connected peer is known to custody) or +/// `"no_custodian"` (nobody connected was found holding the missing columns: +/// none is known to custody them, or every peer asked past the tip answered +/// without them). pub fn inc_data_column_fetch_failure(reason: &str) { LEAN_DATA_COLUMN_FETCH_FAILURES_TOTAL .with_label_values(&[reason]) diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index d47b131be..0117c512a 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -17,6 +17,7 @@ use ethlambda_storage::{Chain, Store}; use libp2p::{PeerId, request_response}; use rand::seq::SliceRandom; use spawned_concurrency::tasks::{Context, send_after}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tracing::{debug, error, info, trace, warn}; @@ -40,7 +41,7 @@ use super::{ messages::{ResponseCode, error_message}, }; use crate::beacon::BeaconWire; -use crate::beacon::column_checks; +use crate::beacon::column_checks::{self, ColumnVerdict}; use crate::beacon::decode::fork_at_slot; use crate::beacon::handler::{self as beacon_handler, StatusVersion}; use crate::beacon::messages::{ @@ -247,11 +248,20 @@ pub async fn handle_req_resp_message( // event: this response is the terminal outcome for // the id either way. match server.outbound_requests.remove(&request_id) { - Some(PendingRequestKind::Columns(block_root, requested)) => { + Some(PendingRequestKind::Columns { + block_root, + columns, + lookup, + }) => { + let request = ColumnRequest { + block_root, + lookup, + peer, + columns, + }; handle_data_column_sidecars_response( - server, peer, block_root, &requested, sidecars, ctx, - ) - .await; + server, request, sidecars, ctx, + ); } Some(PendingRequestKind::ColumnRange { start_slot, @@ -325,7 +335,7 @@ pub async fn handle_req_resp_message( Some( kind @ (PendingRequestKind::Root(_) | PendingRequestKind::Range { .. } - | PendingRequestKind::Columns(..) + | PendingRequestKind::Columns { .. } | PendingRequestKind::ColumnRange { .. }), ) => { error!( @@ -384,7 +394,7 @@ pub async fn handle_req_resp_message( // data column protocol, which answers with // `DataColumnSidecars`, never with `Blocks`. Some( - kind @ (PendingRequestKind::Columns(..) + kind @ (PendingRequestKind::Columns { .. } | PendingRequestKind::ColumnRange { .. } | PendingRequestKind::EnvelopeRoot(_) | PendingRequestKind::EnvelopeRange { .. }), @@ -416,11 +426,21 @@ pub async fn handle_req_resp_message( // forever and deduplicates every later fetch. handle_fetch_failure(server, root, peer, ctx).await; } - Some(PendingRequestKind::Columns(block_root, _)) => { + Some(PendingRequestKind::Columns { + block_root, + columns, + lookup, + }) => { // Same reasoning as the `Root` arm above: an // error response is the whole exchange, so - // this is the only place that can retire it. - handle_column_fetch_failure(server, block_root, peer, ctx).await; + // this is the only place that can settle it. + let request = ColumnRequest { + block_root, + lookup, + peer, + columns, + }; + handle_column_fetch_failure(server, request, ctx); } // Nothing to retire: a range prefetch has no // pending entry and nothing waits on it. A peer @@ -475,8 +495,18 @@ pub async fn handle_req_resp_message( "BlocksByRange request failed; retry is disabled" ); } - Some(PendingRequestKind::Columns(block_root, _)) => { - handle_column_fetch_failure(server, block_root, peer, ctx).await; + Some(PendingRequestKind::Columns { + block_root, + columns, + lookup, + }) => { + let request = ColumnRequest { + block_root, + lookup, + peer, + columns, + }; + handle_column_fetch_failure(server, request, ctx); } // Nothing waits on a range prefetch, so a failure is only // worth a line: the blocks it rode alongside have their own @@ -1039,9 +1069,9 @@ pub(crate) fn record_peer_custody(server: &mut P2PServer, peer: PeerId, custody_ /// The peers known to custody `column`, among those currently connected. /// /// A peer with no entry is omitted rather than assumed: its custody is unknown, -/// and [`fetch_data_columns_from_peer`] falls back to the whole connected set -/// when this comes back empty, so an unknown peer is still reachable — just not -/// preferred over one we know holds the column. +/// and [`fetch_data_columns_from_peer`] asks such a peer for a column no known +/// custodian covers, so an unknown peer is still reachable — just not preferred +/// over one we know holds the column. pub(crate) fn columns_custodied_by(server: &P2PServer, column: u64) -> Vec { server .connected_peers @@ -1060,14 +1090,22 @@ pub(crate) fn columns_custodied_by(server: &P2PServer, column: u64) -> Vec, + missed: &HashSet<(PeerId, u64)>, + retry_missed: bool, ) -> (HashMap>, Vec) { let mut by_peer: HashMap> = HashMap::new(); let mut uncovered = Vec::new(); @@ -1076,13 +1114,15 @@ fn group_columns_by_custody_peer( let candidates: Vec = columns_custodied_by(server, column) .into_iter() .filter(|peer| !exclude.contains(peer)) + .filter(|peer| retry_missed || !missed.contains(&(*peer, column))) .collect(); // `min_by_key` over the load already assigned in this same call, with // the peer id breaking ties so the choice is deterministic for a given // peer set rather than dependent on HashSet iteration order. - let chosen = candidates - .iter() - .min_by_key(|peer| (by_peer.get(*peer).map_or(0, Vec::len), **peer)); + let chosen = candidates.iter().min_by_key(|peer| { + let load = by_peer.get(*peer).map_or(0, Vec::len); + (missed.contains(&(**peer, column)), load, **peer) + }); match chosen { Some(&peer) => by_peer.entry(peer).or_default().push(column), None => uncovered.push(column), @@ -1092,9 +1132,23 @@ fn group_columns_by_custody_peer( (by_peer, uncovered) } -/// Ask a connected peer for specific columns of `block_root`, mirroring -/// [`fetch_block_from_peer`]'s peer selection, dedup and retry bookkeeping -/// against `pending_column_requests` rather than `pending_root_requests`. +/// A fresh id for a column lookup, which its requests carry (see +/// [`PendingColumnRequest::lookup`]). +fn next_column_lookup() -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(1); + NEXT.fetch_add(1, Ordering::Relaxed) +} + +/// Ask connected peers for `columns` of `block_root`, starting a lookup in +/// `pending_column_requests` or adding to the one already there. Returns +/// whether any request went out. +/// +/// Each column goes to a peer known to custody it; see +/// [`plan_column_requests`] for where the rest go. Peers this lookup has +/// written off (`failed_peers`) are not asked. A column with nobody left to +/// go to is not asked at all: the lookup ends once nothing else is open (see +/// [`retire_column_lookup`]), and the chain actor's per-slot re-ask starts a +/// new one against whatever peers it has then. /// /// Beacon-only, unconditionally: unlike [`fetch_block_from_peer`], which has /// to dispatch on `server.wire` because both chains serve a block-shaped @@ -1112,76 +1166,21 @@ pub async fn fetch_data_columns_from_peer( return false; } - // Same pool-narrowing-then-fallback shape as `fetch_block_from_peer`. - let failed = server - .pending_column_requests - .get(&block_root) - .map(|p| &p.failed_peers); - let pool: Vec<_> = if failed.is_none_or(|f| f.is_empty()) { - server.connected_peers.keys().copied().collect() - } else { - let failed = failed.unwrap(); - server - .connected_peers - .keys() - .copied() - .filter(|p| !failed.contains(p)) - .collect() - }; - - let pool = if pool.is_empty() { - debug!(%block_root, "All peers failed for this lookup, retrying with full peer set"); - if let Some(pending) = server.pending_column_requests.get_mut(&block_root) { - pending.failed_peers.clear(); - } - server.connected_peers.keys().copied().collect() - } else { - pool - }; - let excluded = server.connected_peers.len() - pool.len(); - - // Aim each column at a peer that custodies it. Whatever no connected peer - // is known to custody falls back to one random peer from the pool, which - // is all this function could ever do before: it may hold the column and - // simply not have told us its count yet. - let exclude: HashSet = server - .connected_peers - .keys() - .filter(|peer| !pool.contains(peer)) - .copied() - .collect(); - let (mut by_peer, uncovered) = group_columns_by_custody_peer(server, &columns, &exclude); - - if !uncovered.is_empty() { - match pool.choose(&mut rand::thread_rng()) { - Some(&peer) => { - debug!( - %block_root, - %peer, - count = uncovered.len(), - "No connected peer is known to custody these columns; asking one at random" - ); - by_peer.entry(peer).or_default().extend(uncovered); - } - None => { - debug!(%block_root, "Failed to select random peer"); - return false; - } - } - } + let plan = plan_column_requests(server, block_root, &columns); + let excluded = plan.excluded; + let lookup = plan.lookup; - let mut sent_count = 0usize; - for (peer, columns) in by_peer { - let count = columns.len(); - let requested = columns.clone(); - let column_indices = match ColumnIndices::try_from(columns) { + let mut sent: Vec<(PeerId, Vec)> = Vec::new(); + for (peer, peer_columns) in plan.by_peer { + let count = peer_columns.len(); + let column_indices = match ColumnIndices::try_from(peer_columns.clone()) { Ok(indices) => indices, Err(err) => { // A caller asking for more than one block's worth of columns is a // programming error on the chain-actor side, not a peer or wire // fault, so this does not retry. error!(%block_root, ?err, "Too many columns requested in one DataColumnsByRoot lookup"); - return false; + continue; } }; let identifier = DataColumnsByRootIdentifier { @@ -1202,42 +1201,151 @@ pub async fn fetch_data_columns_from_peer( debug!(%block_root, %peer, "Failed to send DataColumnsByRoot request (swarm adapter closed)"); continue; }; - server.outbound_requests.insert( - request_id, - PendingRequestKind::Columns(block_root, requested), - ); - sent_count += 1; + let kind = PendingRequestKind::Columns { + block_root, + columns: peer_columns.clone(), + lookup, + }; + server.outbound_requests.insert(request_id, kind); + sent.push((peer, peer_columns)); } - if sent_count == 0 { + if sent.is_empty() { return false; } - // `or_insert` rather than unconditional insert: a retry re-enters here - // with the same root already tracked, and must not reset `attempts` back - // to a fresh lookup's value. - let pending = - server - .pending_column_requests - .entry(block_root) - .or_insert(PendingColumnRequest { - columns, - attempts: 1, - failed_peers: HashSet::new(), - in_flight: 0, - last_asked: Instant::now(), - }); - // Set rather than added to: every request from the previous round has - // already reported back, since this round only starts once the last one - // did (see `PendingColumnRequest::in_flight`). - pending.in_flight = sent_count; - // Each round's own timestamp, not the lookup's first: a ladder still + // `or_insert_with` rather than an unconditional insert: a retry re-enters + // here with the same root already tracked, and must not reset `attempts` + // back to a fresh lookup's value. A new lookup tracks every column it was + // asked for, sent or not, so one nobody could be asked for still counts + // as missing. + let pending = server + .pending_column_requests + .entry(block_root) + .or_insert_with(|| PendingColumnRequest::new(columns, lookup)); + for (peer, peer_columns) in sent { + if plan.beyond_custody == Some(peer) { + pending.asked_beyond_custody = true; + } + for column in peer_columns { + pending.asked.insert(column, peer); + } + } + // Each send's own timestamp, not the lookup's first: a ladder still // working through the peer set is alive, however long it has been running. pending.last_asked = Instant::now(); true } +/// Which peer [`fetch_data_columns_from_peer`] asks for which column. +struct ColumnPlan { + /// The lookup's id: the existing one's, or a fresh one's. + lookup: u64, + by_peer: HashMap>, + /// How many peers the lookup's history rules out. + excluded: usize, + /// The peer asked for columns outside its known custody, if one is. + beyond_custody: Option, +} + +/// Plan the requests for `columns` of `block_root`'s lookup (a new one when +/// there is none yet). +/// +/// Each column goes to a peer known to custody it. Whatever no known +/// custodian covers goes to one peer whose custody is unknown, the same bet +/// the range path makes (see [`peers_of_unknown_custody`]). +/// +/// A peer that already answered this lookup without a column is asked for it +/// again only while the block is at the tip (its slot is the wall clock's or +/// later), and then after every peer that has not: at the tip it may have +/// received the column since, as `hold_block_for_columns` explains. Past the +/// tip there is no gossip left to bring it, so asking again would only spend +/// the ladder. A lookup that has not read the block's slot yet counts as past +/// the tip. +/// +/// Only when no peer of unknown custody is left do uncovered columns go to a +/// peer that has said what it keeps, and only once per lookup. Such a peer +/// may hold more than it custodies, and asking one at random each slot is all +/// this node did before custody was known; but walking every one of them, a +/// round each, is what a lookup for a column nobody holds used to cost. +fn plan_column_requests(server: &P2PServer, block_root: H256, columns: &[u64]) -> ColumnPlan { + let no_peers = HashSet::new(); + let no_misses = HashSet::new(); + let pending = server.pending_column_requests.get(&block_root); + let lookup = pending.map_or_else(next_column_lookup, |pending| pending.lookup); + let failed = pending.map_or(&no_peers, |pending| &pending.failed_peers); + let missed = pending.map_or(&no_misses, |pending| &pending.missed); + let beyond_custody_spent = pending.is_some_and(|pending| pending.asked_beyond_custody); + let wall_slot = server.wire.beacon().map(BeaconWire::wall_slot); + let at_tip = pending + .and_then(|pending| pending.slot) + .zip(wall_slot) + .is_some_and(|(slot, now)| slot >= now); + + let (mut by_peer, uncovered) = + group_columns_by_custody_peer(server, columns, failed, missed, at_tip); + let mut plan = ColumnPlan { + lookup, + by_peer: HashMap::new(), + excluded: failed.len(), + beyond_custody: None, + }; + if uncovered.is_empty() { + plan.by_peer = by_peer; + return plan; + } + + let misses = |peer: &PeerId| { + uncovered + .iter() + .filter(|&&column| missed.contains(&(*peer, column))) + .count() + }; + let unknown: Vec = peers_of_unknown_custody(server, usize::MAX) + .into_iter() + .filter(|peer| !failed.contains(peer)) + .filter(|peer| at_tip || misses(peer) == 0) + .collect(); + let fewest_misses = unknown.iter().map(misses).min(); + let candidates: Vec = match fewest_misses { + Some(fewest) => unknown + .into_iter() + .filter(|peer| misses(peer) == fewest) + .collect(), + None if !beyond_custody_spent => server + .connected_peers + .keys() + .filter(|peer| !failed.contains(*peer) && misses(peer) == 0) + .copied() + .collect(), + None => Vec::new(), + }; + match candidates.choose(&mut rand::thread_rng()) { + Some(&peer) => { + let custody_known = server.peer_custody.contains_key(&peer); + debug!( + %block_root, + %peer, + custody_known, + count = uncovered.len(), + "No connected peer is known to custody these columns; asking one that might hold them" + ); + if custody_known { + plan.beyond_custody = Some(peer); + } + by_peer.entry(peer).or_default().extend(uncovered); + } + None => debug!( + %block_root, + count = uncovered.len(), + "No connected peer is known to custody these columns, and none is left to ask" + ), + } + plan.by_peer = by_peer; + plan +} + async fn request_next_range_batch(server: &mut P2PServer) -> bool { let Some((peer, batch)) = server .range_sync_state @@ -1554,139 +1662,168 @@ pub(super) fn retire_root_attempt(pending: &mut PendingRequest, peer: PeerId) -> Some(Duration::from_millis(backoff_ms)) } -/// Retire one failed attempt at fetching `block_root`'s missing columns. -/// -/// Mirrors [`handle_fetch_failure`]'s funnel and invariant: every path that -/// ends an attempt must come through here, or `block_root` stays in -/// `pending_column_requests` and `fetch_missing_columns` folds every later -/// call into it rather than starting a second lookup, for the life of the -/// process. -/// -/// `MAX_FETCH_RETRIES` attempts, each bounded by the request-response layer's -/// own per-request timeout and spaced by the doubling backoff below, is the -/// whole bound on how long a lookup persists; see the comment above that -/// constant's definition. A held block is reclaimed by finality on its own -/// schedule regardless, and the ladder stays well inside that window. -async fn handle_column_fetch_failure( - server: &mut P2PServer, - block_root: H256, - peer: PeerId, - ctx: &Context, -) { - let Some(pending) = server.pending_column_requests.get_mut(&block_root) else { - return; - }; +/// One request of a column lookup: which lookup sent it, to whom, and for +/// which columns. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ColumnRequest { + pub(crate) block_root: H256, + /// The lookup that sent it (see [`PendingColumnRequest::lookup`]). + pub(crate) lookup: u64, + pub(crate) peer: PeerId, + /// The columns it asked for. + pub(crate) columns: Vec, +} - let outcome = retire_column_attempt(pending, peer); - follow_column_attempt(server, block_root, peer, outcome, ctx); +/// How one request of a column lookup ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ColumnReport { + pub(crate) request: ColumnRequest, + pub(crate) reply: ColumnReply, } -/// Carry out what [`retire_column_attempt`] or [`settle_column_answer`] -/// decided for `block_root`'s lookup. -fn follow_column_attempt( - server: &mut P2PServer, - block_root: H256, - peer: PeerId, - outcome: ColumnAttemptOutcome, - ctx: &Context, -) { - match outcome { - ColumnAttemptOutcome::RoundIncomplete { in_flight } => { - trace!( - %block_root, - %peer, - in_flight, - "One peer of this column lookup reported back; waiting for the rest of the round" - ); - } - ColumnAttemptOutcome::GiveUp { attempts } => { - error!(%block_root, %peer, attempts, - "Data column fetch failed after max retries, giving up"); - server.pending_column_requests.remove(&block_root); - metrics::inc_data_column_fetch_failure("max_retries"); - } - ColumnAttemptOutcome::Retry { attempts, backoff } => { - debug!(%block_root, %peer, attempts, ?backoff, "Data column lookup still missing columns, scheduling retry"); - send_after( - backoff, - ctx.clone(), - p2p_protocol::RetryDataColumnFetch { block_root }, - ); +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ColumnReply { + /// No answer: the request failed on the wire, timed out, or was refused + /// with an error response. + Failed, + /// An answer, with each sidecar it carried for the request and what the + /// chain checks made of it. Empty when it carried none. + Answered(Vec<(u64, ColumnVerdict)>), +} + +impl ColumnReport { + /// The requested columns whose sidecars passed the checks, or were already + /// stored. + fn delivered(&self) -> Vec { + let ColumnReply::Answered(verdicts) = &self.reply else { + return Vec::new(); + }; + verdicts + .iter() + .filter(|(column, verdict)| { + *verdict == ColumnVerdict::Delivered && self.request.columns.contains(column) + }) + .map(|(column, _)| *column) + .collect() + } + + /// Whether the peer is not asked again in this lookup: its request failed, + /// or it sent a sidecar the checks refused. + /// + /// An answer without a column is not held against the peer, whether it + /// was asked as a known custodian or on the chance that it held the + /// column. The specification lets a peer answer with what it has, and at + /// the tip a peer often has not received the column yet, which is why the + /// chain actor waits a slot before asking for a block still at the tip + /// (see `hold_block_for_columns`). Such a peer is recorded in `missed` + /// instead, and asked for that column again only after every peer that + /// has not missed it. + fn peer_at_fault(&self) -> bool { + match &self.reply { + ColumnReply::Failed => true, + ColumnReply::Answered(verdicts) => verdicts + .iter() + .any(|(_, verdict)| *verdict == ColumnVerdict::Rejected), } } } -/// What a failed request means for the lookup it belonged to. +/// What one request reporting back means for its lookup. #[derive(Debug, PartialEq, Eq)] enum ColumnAttemptOutcome { - /// Other requests from this same attempt are still open, so the attempt - /// has no outcome yet. - RoundIncomplete { in_flight: usize }, - /// The ladder is exhausted; the caller retires the lookup. - GiveUp { attempts: u32 }, - /// The attempt is spent and another is due after `backoff`. + /// Every column the lookup wanted has arrived. The lookup is retired. + Complete, + /// Nothing to ask for now: every column still missing is on the wire with + /// another request (`asked` columns are), or a retry already due will ask + /// for it. + Waiting { asked: usize }, + /// Ask for whatever no open request covers after `backoff`: zero, and no + /// attempt spent, when the request delivered everything it asked for. Retry { attempts: u32, backoff: Duration }, + /// The ladder is exhausted. The lookup is retired. + GiveUp { attempts: u32 }, } -/// Charge `peer`'s failure against `pending` and say what follows. -/// -/// Split out from [`handle_column_fetch_failure`] so the round arithmetic can -/// be tested without an actor context: everything the decision depends on is in -/// `pending`, and everything it causes (the store mutation, the metric, the -/// timer) is the caller's. +/// Settle one request of `pending` and say what follows. /// -/// One attempt is one round, however many peers it was split across. While any -/// request of the round is still open, a sibling's failure is not the attempt's -/// outcome: the round may still be answered, and treating each failure as its -/// own attempt would burn the ladder several times faster *and* schedule one -/// fan-out per failure, which squares with every retry. -fn retire_column_attempt(pending: &mut PendingColumnRequest, peer: PeerId) -> ColumnAttemptOutcome { - pending.failed_peers.insert(peer); - close_column_request(pending) -} - -/// Take a non-empty answer's columns off what `pending` still asks for, and -/// say what follows: `None` once nothing is left, for the caller to retire the -/// lookup. +/// Split out from [`report_column_request`] so the arithmetic can be tested +/// without a server: everything the decision depends on is in `pending` and +/// the report. /// -/// A lookup is split across the peers that custody its columns, so one peer's -/// answer is often only part of it. Retiring the lookup on that first answer -/// left every column asked of another peer in the same round to wait for the -/// chain actor's next per-slot re-drive, however soon it could have come: on a -/// devnet, about one gloas envelope in twenty waited ~11.7 s for its columns. -/// A short answer now ends its request the way a failure does, so the rest is -/// asked again on the same ladder. +/// A lookup is split across the peers that custody its columns, so each +/// request settles its own columns only: what it delivered leaves the lookup, +/// and what it left out is asked again at once (after the backoff), without +/// waiting for the lookup's other requests, which may be waiting out a request +/// timeout. A lookup used to be retired on its first non-empty answer, which +/// left every column asked of another peer to wait for the chain actor's next +/// per-slot re-ask: on a devnet, about one gloas envelope in twenty waited +/// ~11.7 s for its columns. /// -/// Charged to `peer` only if it left out a column it was itself asked for. A -/// peer that sent its whole share did nothing wrong: what remains is another -/// request's, or a column merged into the lookup after this request went out. -fn settle_column_answer( +/// Only a request that left out a column it was asked for spends an attempt +/// and doubles the backoff. A request that delivered its whole share spent +/// nothing: what is left is another request's, or a column merged into the +/// lookup after it was sent, which is asked for at once. However many of a +/// lookup's requests come back short, one retry is due at a time, and it asks +/// for all of their leftovers. +fn settle_column_report( pending: &mut PendingColumnRequest, - peer: PeerId, - requested: &[u64], - delivered: &[u64], -) -> Option { + report: &ColumnReport, +) -> ColumnAttemptOutcome { + let request = &report.request; + let delivered = report.delivered(); pending.columns.retain(|column| !delivered.contains(column)); - if pending.columns.is_empty() { - return None; + + // A request of a lookup this one replaced: its columns are here all the + // same, but it is not one of this lookup's requests, so it settles nothing + // else. + if request.lookup != pending.lookup { + return if pending.columns.is_empty() { + ColumnAttemptOutcome::Complete + } else { + ColumnAttemptOutcome::Waiting { + asked: pending.asked.len(), + } + }; } - if requested.iter().any(|column| !delivered.contains(column)) { - pending.failed_peers.insert(peer); + + for column in &request.columns { + if pending.asked.get(column) == Some(&request.peer) { + pending.asked.remove(column); + } + } + if pending.columns.is_empty() { + return ColumnAttemptOutcome::Complete; } - Some(close_column_request(pending)) -} -/// One request of `pending`'s round has reported back without completing the -/// lookup: the round's outcome once it was the last one open. -fn close_column_request(pending: &mut PendingColumnRequest) -> ColumnAttemptOutcome { - pending.in_flight = pending.in_flight.saturating_sub(1); + let left_out: Vec = request + .columns + .iter() + .copied() + .filter(|column| !delivered.contains(column)) + .collect(); + for &column in &left_out { + pending.missed.insert((request.peer, column)); + } + if report.peer_at_fault() { + pending.failed_peers.insert(request.peer); + } - if pending.in_flight > 0 { - return ColumnAttemptOutcome::RoundIncomplete { - in_flight: pending.in_flight, + if pending.retry_due || pending.unasked().is_empty() { + return ColumnAttemptOutcome::Waiting { + asked: pending.asked.len(), }; } + let missed = left_out + .iter() + .any(|column| pending.columns.contains(column)); + if !missed { + pending.retry_due = true; + return ColumnAttemptOutcome::Retry { + attempts: pending.attempts, + backoff: Duration::ZERO, + }; + } if pending.attempts >= MAX_FETCH_RETRIES { return ColumnAttemptOutcome::GiveUp { attempts: pending.attempts, @@ -1696,13 +1833,245 @@ fn close_column_request(pending: &mut PendingColumnRequest) -> ColumnAttemptOutc let backoff_ms = INITIAL_BACKOFF_MS * BACKOFF_MULTIPLIER.pow(pending.attempts - 1); let attempts = pending.attempts; pending.attempts += 1; - + pending.retry_due = true; ColumnAttemptOutcome::Retry { attempts, backoff: Duration::from_millis(backoff_ms), } } +/// Settle `report` against its lookup, retiring the lookup when it is complete +/// or its ladder has run out. `None` when no lookup is waiting on this root: +/// it completed, gave up, or ran out of peers to ask before this request +/// reported back. +/// +/// Every request of a lookup must report through here exactly once, or its +/// columns stay on the wire for good and are never asked again until the +/// lookup goes stale (see `STALE_COLUMN_LOOKUP`). +fn report_column_request( + server: &mut P2PServer, + report: &ColumnReport, +) -> Option { + let block_root = report.request.block_root; + let pending = server.pending_column_requests.get_mut(&block_root)?; + let outcome = settle_column_report(pending, report); + match outcome { + ColumnAttemptOutcome::Complete => { + retire_column_lookup(server, block_root, Retirement::Complete); + } + ColumnAttemptOutcome::GiveUp { attempts } => { + // A ladder spent only on columns no connected peer is known to + // keep has found nobody holding them, which is not a fault of + // this node's or of any peer's. + let uncovered_only = + server + .pending_column_requests + .get(&block_root) + .is_some_and(|pending| { + pending + .columns + .iter() + .all(|&column| columns_custodied_by(server, column).is_empty()) + }); + let why = if uncovered_only { + Retirement::NobodyToAsk + } else { + let peer = report.request.peer; + Retirement::GaveUp { attempts, peer } + }; + retire_column_lookup(server, block_root, why); + } + ColumnAttemptOutcome::Waiting { .. } | ColumnAttemptOutcome::Retry { .. } => {} + } + Some(outcome) +} + +/// Settle one request of a column lookup and schedule what follows: a failed +/// one from the request-response layer, an answer once its sidecars have been +/// through the checks. +pub(crate) fn settle_column_request( + server: &mut P2PServer, + report: ColumnReport, + ctx: &Context, +) { + if let Some(outcome) = report_column_request(server, &report) { + follow_column_attempt(&report.request, &outcome, ctx); + } +} + +/// A column request that got no answer: failed on the wire, timed out, or was +/// refused with an error response. +fn handle_column_fetch_failure( + server: &mut P2PServer, + request: ColumnRequest, + ctx: &Context, +) { + let report = ColumnReport { + request, + reply: ColumnReply::Failed, + }; + settle_column_request(server, report, ctx); +} + +/// Schedule what [`report_column_request`] decided. A retired lookup has +/// nothing left to schedule: [`retire_column_lookup`] already said why. +fn follow_column_attempt( + request: &ColumnRequest, + outcome: &ColumnAttemptOutcome, + ctx: &Context, +) { + let block_root = request.block_root; + let peer = request.peer; + match *outcome { + ColumnAttemptOutcome::Waiting { asked } => { + trace!( + %block_root, + %peer, + asked, + "A data column request reported back; the rest of the lookup is still on the wire" + ); + } + ColumnAttemptOutcome::Retry { attempts, backoff } => { + debug!(%block_root, %peer, attempts, ?backoff, "Data column lookup still missing columns, scheduling retry"); + let retry = p2p_protocol::RetryDataColumnFetch { + block_root, + lookup: request.lookup, + }; + send_after(backoff, ctx.clone(), retry); + } + ColumnAttemptOutcome::Complete | ColumnAttemptOutcome::GiveUp { .. } => {} + } +} + +/// Why a column lookup ended. +#[derive(Debug, PartialEq, Eq)] +enum Retirement { + /// Every column it wanted has arrived. + Complete, + /// The ladder ran out, ending on a request to `peer`. + GaveUp { attempts: u32, peer: PeerId }, + /// Nobody connected was found holding what is still missing: nothing is + /// on the wire and no peer is left to ask (none is known to custody it, + /// or every one asked past the tip answered without it), or the ladder + /// ran out on columns no connected peer is known to custody. + NobodyToAsk, + /// Nothing is on the wire, and nothing is connected at all. + NoPeers, +} + +/// Retire `block_root`'s lookup, saying why. The one place a lookup ends; +/// `fetch_missing_columns` also drops a stale one to start its replacement. +/// +/// A lookup that ends short of its columns is not lost for good: the chain +/// actor asks again on every slot tick for every block or envelope still +/// held, and that ask starts a new lookup against whatever peers this node +/// has by then. +fn retire_column_lookup(server: &mut P2PServer, block_root: H256, why: Retirement) { + let Some(pending) = server.pending_column_requests.remove(&block_root) else { + return; + }; + match why { + Retirement::Complete => { + trace!(%block_root, "Every column of this data column lookup has arrived"); + } + Retirement::GaveUp { attempts, peer } => { + error!(%block_root, %peer, attempts, + "Data column fetch failed after max retries, giving up"); + metrics::inc_data_column_fetch_failure("max_retries"); + } + // Not an error, unlike running out of retries: the columns are not + // anywhere this node can reach yet, and the next re-ask may find a + // peer that connected since. + Retirement::NobodyToAsk => { + let missing = pending.columns; + debug!( + %block_root, + ?missing, + "No connected peer is left to ask for these columns; leaving them to the next re-ask" + ); + metrics::inc_data_column_fetch_failure("no_custodian"); + } + // Counted as `no_peers` where the send found nobody connected. + Retirement::NoPeers => { + debug!(%block_root, "No connected peers to ask for this lookup's columns"); + } + } +} + +/// The retry a column lookup scheduled for what its requests left out. +pub(crate) async fn retry_column_lookup(server: &mut P2PServer, block_root: H256, lookup: u64) { + let Some(pending) = server + .pending_column_requests + .get_mut(&block_root) + .filter(|pending| pending.lookup == lookup) + else { + trace!(%block_root, "Data column lookup ended during backoff, skipping retry"); + return; + }; + pending.retry_due = false; + ask_unasked_columns(server, block_root).await; +} + +/// Ask for the columns of `block_root`'s lookup that no open request covers. +/// +/// The store is read first: gossip often delivers a column while a lookup +/// started at the tip is still asking for it, and asking again would download +/// a sidecar the checks then drop as already stored. When that leaves nothing +/// missing, the lookup is complete. When something is missing but none of it +/// can be asked for and nothing is on the wire, the lookup ends: nothing would +/// ever ask for it again within it. +pub(crate) async fn ask_unasked_columns(server: &mut P2PServer, block_root: H256) { + drop_stored_columns(server, block_root); + let Some(pending) = server.pending_column_requests.get(&block_root) else { + return; + }; + if pending.columns.is_empty() { + retire_column_lookup(server, block_root, Retirement::Complete); + return; + } + let unasked = pending.unasked(); + if unasked.is_empty() { + return; + } + let sent = fetch_data_columns_from_peer(server, block_root, unasked).await; + let open = server + .pending_column_requests + .get(&block_root) + .is_some_and(|pending| !pending.asked.is_empty()); + if !sent && !open { + let why = if server.connected_peers.is_empty() { + Retirement::NoPeers + } else { + Retirement::NobodyToAsk + }; + retire_column_lookup(server, block_root, why); + } +} + +/// Take off `block_root`'s lookup the columns the store already holds. +/// +/// The block's slot is read once per lookup, and only here: on beacon, +/// [`Store::block_entry`] decodes the whole block. A block the store does not +/// have leaves the lookup as it is. +fn drop_stored_columns(server: &mut P2PServer, block_root: H256) { + let store = &server.store; + let Some(pending) = server.pending_column_requests.get_mut(&block_root) else { + return; + }; + let slot = match pending.slot { + Some(slot) => slot, + None => { + let Some((slot, _)) = store.block_entry(&block_root) else { + return; + }; + pending.slot = Some(slot); + slot + } + }; + pending + .columns + .retain(|&column| !store.has_data_column(slot, &block_root, column)); +} /// The beacon wire, or a refusal sent on `channel`. /// /// A beacon request reaching a lean node means a peer negotiated a protocol @@ -2378,7 +2747,8 @@ pub(crate) async fn request_beacon_data_columns_by_range( // No exclusions: unlike a by-root retry, this has no failed-peer history to // avoid, and a peer that cannot answer simply returns nothing. - let (mut by_peer, uncovered) = group_columns_by_custody_peer(server, &columns, &HashSet::new()); + let (mut by_peer, uncovered) = + group_columns_by_custody_peer(server, &columns, &HashSet::new(), &HashSet::new(), true); if !uncovered.is_empty() { // "Uncovered" means no peer is *known* to custody these, and custody is // only known once a peer's `metadata/3` answer or its ENR `cgc` has @@ -2480,27 +2850,6 @@ pub async fn request_beacon_block_by_root( Some(request_id) } -/// Take delivery of a `DataColumnsByRoot` response and run every sidecar it -/// carried through the chain checks (`beacon::column_checks`), which send the -/// chain actor the ones that pass. The specification requires a sidecar -/// obtained by any other means to be treated as if it had arrived on gossip; -/// the chain checks are gossip's rules less the two that only mean something -/// on a topic (the subnet and the seen cache), and the chain actor keeps what -/// reaches it without checking it again, so they are the only checks a -/// fetched sidecar gets. -/// -/// Unlike a block-by-root answer, which names exactly one document that -/// either matches the requested root or doesn't, the specification permits a -/// peer to return fewer sidecars than were asked for ("Clients MUST respond -/// with at least one sidecar, if they have it" — not with every one), so a -/// non-empty answer is treated as this lookup's success regardless of -/// whether it is the full requested set. Whatever is still missing is the -/// next task's concern: it is driven by a fresh `fetch_block` call naming -/// what is still missing once the held block is re-checked, not by this -/// function retrying for the remainder. Zero sidecars is the one case that is -/// not success: it means this peer had none of what was asked, and is charged -/// as a failed attempt the same way an empty `BlocksByRoot` answer is, backed -/// off and retried against another peer. /// Take delivery of a `DataColumnsByRange` prefetch and forward what it carries. /// /// No failure handling, unlike the by-root path: nothing is waiting on this @@ -2573,18 +2922,66 @@ fn retain_requested_columns( (kept, dropped) } -async fn handle_data_column_sidecars_response( +/// Take delivery of a `DataColumnsByRoot` response and run every sidecar it +/// carried through the chain checks (`beacon::column_checks`), which send the +/// chain actor the ones that pass. The specification requires a sidecar +/// obtained by any other means to be treated as if it had arrived on gossip; +/// the chain checks are gossip's rules less the two that only mean something +/// on a topic (the subnet and the seen cache), and the chain actor keeps what +/// reaches it without checking it again, so they are the only checks a +/// fetched sidecar gets. +/// +/// The request's lookup is settled once the checks are done, on what passed +/// them ([`settle_column_request`], through the actor's own mailbox), not on +/// what the peer sent: a sidecar with the right root and index can still fail +/// them, and a gloas one carries no signature. Until then its columns count +/// as still on the wire, so they are not asked of anyone else meanwhile. +/// +/// Unlike a block-by-root answer, which names exactly one document that +/// either matches the requested root or doesn't, the specification permits a +/// peer to return fewer sidecars than were asked for ("Clients MUST respond +/// with at least one sidecar, if they have it" — not with every one). So a +/// short answer settles what it brought, and the rest is asked again (see +/// [`settle_column_report`]); zero sidecars is the shortest answer, settled +/// at once since there is nothing to check. +/// +/// The sidecars are checked and forwarded whether or not a lookup is still +/// waiting on them: one that already completed, gave up or was replaced has +/// no use for them, but the chain actor may. +fn handle_data_column_sidecars_response( server: &mut P2PServer, - peer: PeerId, - block_root: H256, - requested: &[u64], + request: ColumnRequest, sidecars: Vec, ctx: &Context, ) { + let back = ctx.clone(); + let on_checked = move |report: ColumnReport| { + let message = p2p_protocol::ColumnAnswerChecked { report }; + let _ = back + .send(message) + .inspect_err(|err| warn!(%err, "Failed to settle a checked DataColumnsByRoot answer")); + }; + if let Some(outcome) = take_column_answer(server, request.clone(), sidecars, on_checked) { + follow_column_attempt(&request, &outcome, ctx); + } +} + +/// [`handle_data_column_sidecars_response`] without the actor: keep what +/// answers `request`, then either settle an answer with nothing left at once, +/// or hand the rest to the checks, which call `on_checked` with the report to +/// settle once they are done. Returns the outcome of a request settled here. +fn take_column_answer( + server: &mut P2PServer, + request: ColumnRequest, + sidecars: Vec, + on_checked: impl FnOnce(ColumnReport) + Send + 'static, +) -> Option { + let block_root = request.block_root; + let peer = request.peer; let received = sidecars.len(); trace!(%peer, %block_root, received, "Received DataColumnsByRoot response"); - let (sidecars, unrequested) = retain_requested_columns(sidecars, block_root, requested); + let (sidecars, unrequested) = retain_requested_columns(sidecars, block_root, &request.columns); if unrequested > 0 { debug!( %peer, @@ -2596,33 +2993,24 @@ async fn handle_data_column_sidecars_response( if sidecars.is_empty() { debug!(%peer, %block_root, "DataColumnsByRoot response carried no sidecars"); - handle_column_fetch_failure(server, block_root, peer, ctx).await; - return; - } - - // The lookup is done once every column it asks for has arrived; a short - // answer leaves it open for the rest (see `settle_column_answer`). No entry - // means a sibling's answer already completed it, or the ladder gave up: - // the sidecars are still forwarded below. - let delivered: Vec = sidecars.iter().map(|sidecar| sidecar.index()).collect(); - let settled = server - .pending_column_requests - .get_mut(&block_root) - .map(|pending| settle_column_answer(pending, peer, requested, &delivered)); - match settled { - Some(None) => { - server.pending_column_requests.remove(&block_root); - } - Some(Some(outcome)) => follow_column_attempt(server, block_root, peer, outcome, ctx), - None => {} + let report = ColumnReport { + request, + reply: ColumnReply::Answered(Vec::new()), + }; + return report_column_request(server, &report); } if server.blockchain.is_none() { debug!(%peer, %block_root, "No blockchain handler available"); - return; } - - column_checks::check_and_forward(server, sidecars); + let done: column_checks::Done = Box::new(move |verdicts| { + on_checked(ColumnReport { + request, + reply: ColumnReply::Answered(verdicts), + }); + }); + column_checks::check_and_forward_then(server, sidecars, Some(done)); + None } /// Take delivery of a range answer, checking it against what was asked for, @@ -2740,8 +3128,18 @@ async fn retire_unanswered( match kind { PendingRequestKind::Root(root) => handle_fetch_failure(server, root, peer, ctx).await, PendingRequestKind::Range { .. } => fail_range_request(server, &peer), - PendingRequestKind::Columns(block_root, _) => { - handle_column_fetch_failure(server, block_root, peer, ctx).await; + PendingRequestKind::Columns { + block_root, + columns, + lookup, + } => { + let request = ColumnRequest { + block_root, + lookup, + peer, + columns, + }; + handle_column_fetch_failure(server, request, ctx); } PendingRequestKind::ColumnRange { .. } => {} PendingRequestKind::EnvelopeRoot(block_root) => { @@ -2946,11 +3344,13 @@ pub(crate) mod tests { server.peer_custody.insert(holder_of_2, vec![2]); server.peer_custody.insert(holder_of_both, vec![1, 2]); - let (by_peer, uncovered) = group_columns_by_custody_peer(&server, &[1, 2], &HashSet::new()); + let (by_peer, uncovered) = + group_columns_by_custody_peer(&server, &[1, 2], &HashSet::new(), &HashSet::new(), true); // Column 3 is nobody's, so it is reported rather than pinned on a peer // that would answer empty. - let (_, uncovered_3) = group_columns_by_custody_peer(&server, &[3], &HashSet::new()); + let (_, uncovered_3) = + group_columns_by_custody_peer(&server, &[3], &HashSet::new(), &HashSet::new(), true); assert_eq!(uncovered_3, vec![3]); assert!(uncovered.is_empty()); @@ -2975,7 +3375,8 @@ pub(crate) mod tests { // "custodies everything": the caller has its own random fallback for // this, and silently treating unknown as a holder would put us back to // asking arbitrary peers for specific columns. - let (by_peer, uncovered) = group_columns_by_custody_peer(&server, &[7], &HashSet::new()); + let (by_peer, uncovered) = + group_columns_by_custody_peer(&server, &[7], &HashSet::new(), &HashSet::new(), true); assert!(by_peer.is_empty()); assert_eq!(uncovered, vec![7]); @@ -3080,160 +3481,674 @@ pub(crate) mod tests { // The exclusion set is how a retry avoids the peer that just failed; // custody must not override it, or a retry would loop on one peer. + let (by_peer, uncovered) = group_columns_by_custody_peer( + &server, + &[4], + &HashSet::from([failed]), + &HashSet::new(), + true, + ); + + assert!(by_peer.is_empty()); + assert_eq!(uncovered, vec![4]); + } + + #[tokio::test] + async fn a_custodian_that_missed_a_column_is_asked_after_the_others() { + let mut server = unconnected_server().await; + let first = PeerId::random(); + let second = PeerId::random(); + for peer in [first, second] { + server + .connected_peers + .insert(peer, ConnectionDirection::Inbound); + server.peer_custody.insert(peer, vec![4]); + } + let (asked_first, _) = + group_columns_by_custody_peer(&server, &[4], &HashSet::new(), &HashSet::new(), true); + let tried = *asked_first.keys().next().expect("a custodian is chosen"); + let other = if tried == first { second } else { first }; + + // A retry tries the custodian that has not missed this column yet. + let missed = HashSet::from([(tried, 4)]); + let (by_peer, _) = + group_columns_by_custody_peer(&server, &[4], &HashSet::new(), &missed, true); + assert_eq!(by_peer, HashMap::from([(other, vec![4])])); + + // And comes back to one that has, rather than to nobody: at the tip a + // custodian may simply not have had the column yet. + let missed = HashSet::from([(first, 4), (second, 4)]); let (by_peer, uncovered) = - group_columns_by_custody_peer(&server, &[4], &HashSet::from([failed])); + group_columns_by_custody_peer(&server, &[4], &HashSet::new(), &missed, true); + assert_eq!(by_peer.values().flatten().count(), 1); + assert!(uncovered.is_empty()); + // Past the tip nothing is still arriving, so it does not. + let (by_peer, uncovered) = + group_columns_by_custody_peer(&server, &[4], &HashSet::new(), &missed, false); assert!(by_peer.is_empty()); assert_eq!(uncovered, vec![4]); } + /// The id of a lookup built by hand. `next_column_lookup` starts at 1, so + /// no lookup started through `fetch_data_columns_from_peer` has it. + const LOOKUP: u64 = 0; + + const FIRST_BACKOFF: Duration = Duration::from_millis(INITIAL_BACKOFF_MS); + + fn column_lookup(columns: Vec, asked: &[(u64, PeerId)]) -> PendingColumnRequest { + let mut pending = PendingColumnRequest::new(columns, LOOKUP); + pending.asked = asked.iter().copied().collect(); + pending + } + + fn column_request(block_root: H256, peer: PeerId, columns: &[u64]) -> ColumnRequest { + ColumnRequest { + block_root, + lookup: LOOKUP, + peer, + columns: columns.to_vec(), + } + } + + fn column_report(peer: PeerId, columns: &[u64], reply: ColumnReply) -> ColumnReport { + ColumnReport { + request: column_request(H256::repeat_byte(0xC0), peer, columns), + reply, + } + } + + /// An answer carrying `columns`, every one of which passed the checks. + fn delivering(columns: &[u64]) -> ColumnReply { + let verdicts = columns + .iter() + .map(|&column| (column, ColumnVerdict::Delivered)) + .collect(); + ColumnReply::Answered(verdicts) + } + #[test] - fn a_split_lookup_spends_one_attempt_however_many_peers_it_used() { + fn failures_across_a_split_lookup_share_one_retry() { // Three requests open for one root, the shape custody-aware splitting - // produces. Before `in_flight`, each failure was its own attempt and - // scheduled its own fan-out, so one unanswered lookup against eight - // custodians became eight retries, then sixty-four. - let mut pending = PendingColumnRequest { - columns: vec![1, 2, 3], - attempts: 1, - failed_peers: HashSet::new(), - in_flight: 3, - last_asked: Instant::now(), - }; - + // produces. Each failure used to be able to schedule its own fan-out, + // so one unanswered lookup against eight custodians became eight + // retries, then sixty-four. The first failure's leftover is asked + // after the first backoff; the others ride on that retry. + let [a, b, c] = [PeerId::random(), PeerId::random(), PeerId::random()]; + let mut pending = column_lookup(vec![1, 2, 3], &[(1, a), (2, b), (3, c)]); + + let failed = column_report(a, &[1], ColumnReply::Failed); assert_eq!( - retire_column_attempt(&mut pending, PeerId::random()), - ColumnAttemptOutcome::RoundIncomplete { in_flight: 2 } + settle_column_report(&mut pending, &failed), + ColumnAttemptOutcome::Retry { + attempts: 1, + backoff: FIRST_BACKOFF, + } ); + let failed = column_report(b, &[2], ColumnReply::Failed); assert_eq!( - retire_column_attempt(&mut pending, PeerId::random()), - ColumnAttemptOutcome::RoundIncomplete { in_flight: 1 } + settle_column_report(&mut pending, &failed), + ColumnAttemptOutcome::Waiting { asked: 1 } ); - assert_eq!(pending.attempts, 1, "the round's attempt is not spent yet"); - - // Only the last failure of the round is the attempt's outcome, and it - // schedules exactly one retry. + let failed = column_report(c, &[3], ColumnReply::Failed); assert_eq!( - retire_column_attempt(&mut pending, PeerId::random()), - ColumnAttemptOutcome::Retry { - attempts: 1, - backoff: Duration::from_millis(INITIAL_BACKOFF_MS), - } + settle_column_report(&mut pending, &failed), + ColumnAttemptOutcome::Waiting { asked: 0 } ); - assert_eq!(pending.attempts, 2); + + assert_eq!(pending.attempts, 2, "one attempt for the three failures"); + assert_eq!(pending.failed_peers, HashSet::from([a, b, c])); + assert_eq!(pending.unasked(), vec![1, 2, 3], "all for the one retry"); } #[test] fn a_column_lookup_gives_up_once_the_ladder_is_exhausted() { - let mut pending = PendingColumnRequest { - columns: vec![1], - attempts: MAX_FETCH_RETRIES, - failed_peers: HashSet::new(), - in_flight: 1, - last_asked: Instant::now(), - }; + let peer = PeerId::random(); + let mut pending = column_lookup(vec![1], &[(1, peer)]); + pending.attempts = MAX_FETCH_RETRIES; + let failed = column_report(peer, &[1], ColumnReply::Failed); assert_eq!( - retire_column_attempt(&mut pending, PeerId::random()), + settle_column_report(&mut pending, &failed), ColumnAttemptOutcome::GiveUp { attempts: MAX_FETCH_RETRIES } ); } - fn column_lookup(columns: Vec, in_flight: usize) -> PendingColumnRequest { - PendingColumnRequest { - columns, - attempts: 1, - failed_peers: HashSet::new(), - in_flight, - last_asked: Instant::now(), - } - } - #[test] fn an_answer_carrying_every_missing_column_completes_the_lookup() { - let mut pending = column_lookup(vec![1, 2], 1); let peer = PeerId::random(); + let mut pending = column_lookup(vec![1, 2], &[(1, peer), (2, peer)]); + let answer = column_report(peer, &[1, 2], delivering(&[1, 2])); assert_eq!( - settle_column_answer(&mut pending, peer, &[1, 2], &[1, 2]), - None + settle_column_report(&mut pending, &answer), + ColumnAttemptOutcome::Complete ); assert!(pending.failed_peers.is_empty()); } #[test] - fn a_short_answer_keeps_the_lookup_and_retries_the_rest() { - // The lookup used to be retired by any non-empty answer, leaving the - // columns this peer left out for the chain actor's next per-slot - // re-drive. - let mut pending = column_lookup(vec![1, 2, 3], 1); - let peer = PeerId::random(); + fn a_short_answer_has_its_leftover_asked_without_waiting_for_its_siblings() { + // A custodian of column 1 whose request hangs, and a peer of unknown + // custody asked for 2 and 3 that answers with 2 only: the devnet's + // shape. The leftover used to wait out the hung request's 10 s + // timeout before anything asked for it again. + let custodian = PeerId::random(); + let other = PeerId::random(); + let mut pending = column_lookup(vec![1, 2, 3], &[(1, custodian), (2, other), (3, other)]); + let short = column_report(other, &[2, 3], delivering(&[2])); assert_eq!( - settle_column_answer(&mut pending, peer, &[1, 2, 3], &[1]), - Some(ColumnAttemptOutcome::Retry { + settle_column_report(&mut pending, &short), + ColumnAttemptOutcome::Retry { attempts: 1, - backoff: Duration::from_millis(INITIAL_BACKOFF_MS), - }) + backoff: FIRST_BACKOFF, + } + ); + assert_eq!(pending.columns, vec![1, 3]); + assert_eq!( + pending.unasked(), + vec![3], + "1 is still on the wire with the custodian" ); - assert_eq!(pending.columns, vec![2, 3], "only what is still missing"); assert!( - pending.failed_peers.contains(&peer), - "it left out a column it was asked for" + pending.failed_peers.is_empty(), + "at the tip it may not have had 3 yet" ); + assert!(pending.missed.contains(&(other, 3))); } #[test] - fn a_split_lookup_waits_for_every_share_and_charges_only_the_short_one() { - // Custody-aware splitting: a custodian of column 1 and a random peer - // for 2 and 3, the shape a node gets when no peer custodies most of - // its columns. The custodian answering first must not retire the - // lookup the other request still serves. - let mut pending = column_lookup(vec![1, 2, 3], 2); + fn a_custodian_answering_without_its_own_column_is_not_written_off() { + // At the tip a custodian often has not received every column yet. It + // used to be excluded for the rest of the lookup, leaving its column + // to peers known not to keep it. let custodian = PeerId::random(); - let other = PeerId::random(); + let mut pending = column_lookup(vec![1, 2], &[(1, custodian), (2, custodian)]); + let short = column_report(custodian, &[1, 2], delivering(&[1])); assert_eq!( - settle_column_answer(&mut pending, custodian, &[1], &[1]), - Some(ColumnAttemptOutcome::RoundIncomplete { in_flight: 1 }) + settle_column_report(&mut pending, &short), + ColumnAttemptOutcome::Retry { + attempts: 1, + backoff: FIRST_BACKOFF, + } ); + assert!(pending.failed_peers.is_empty()); assert!( - pending.failed_peers.is_empty(), - "the custodian sent its whole share" + pending.missed.contains(&(custodian, 2)), + "another custodian of 2 is tried first, when there is one" ); + } + + #[test] + fn a_request_that_delivered_its_whole_share_spends_no_attempt() { + // Column 4 merged into the lookup after the request for 1 went out. + // Nobody failed, so the retry that asks for it is due at once and + // spends nothing. + let peer = PeerId::random(); + let mut pending = column_lookup(vec![1, 4], &[(1, peer)]); + let whole = column_report(peer, &[1], delivering(&[1])); assert_eq!( - settle_column_answer(&mut pending, other, &[2, 3], &[2]), - Some(ColumnAttemptOutcome::Retry { + settle_column_report(&mut pending, &whole), + ColumnAttemptOutcome::Retry { attempts: 1, - backoff: Duration::from_millis(INITIAL_BACKOFF_MS), - }) + backoff: Duration::ZERO, + } ); - assert_eq!(pending.columns, vec![3]); - assert_eq!(pending.failed_peers, HashSet::from([other])); + assert_eq!(pending.columns, vec![4]); + assert!(pending.failed_peers.is_empty()); + + // However many such rounds a lookup takes, one column delivered and + // another merged in each time, none of them reaches the end of the + // ladder. + let rounds = 2 * u64::from(MAX_FETCH_RETRIES); + for column in 4..4 + rounds { + pending.retry_due = false; + pending.asked.insert(column, peer); + pending.columns.push(column + 1); + let whole = column_report(peer, &[column], delivering(&[column])); + assert_eq!( + settle_column_report(&mut pending, &whole), + ColumnAttemptOutcome::Retry { + attempts: 1, + backoff: Duration::ZERO, + } + ); + } + assert_eq!(pending.attempts, 1); } #[test] - fn a_whole_share_still_retries_a_column_merged_in_after_the_request() { - // `fetch_missing_columns` merges a re-ask's new column into the lookup - // in flight; the request on the wire cannot carry it, so the round's - // end has to ask for it. - let mut pending = column_lookup(vec![1, 4], 1); - let peer = PeerId::random(); + fn a_report_from_a_replaced_lookup_settles_nothing_of_the_new_one() { + // `STALE_COLUMN_LOOKUP` (8 s) is shorter than the request timeout + // (10 s), so a request of a replaced lookup can report after its + // replacement has started. Its failure used to close one of the new + // lookup's requests early. + let new_peer = PeerId::random(); + let old_peer = PeerId::random(); + let mut pending = column_lookup(vec![1, 2], &[(1, new_peer), (2, new_peer)]); + let mut late = column_report(old_peer, &[1, 2], ColumnReply::Failed); + late.request.lookup = LOOKUP + 1; + + assert_eq!( + settle_column_report(&mut pending, &late), + ColumnAttemptOutcome::Waiting { asked: 2 } + ); + assert_eq!( + pending.asked.len(), + 2, + "the new lookup's requests stay open" + ); + assert_eq!(pending.attempts, 1); + assert!(pending.failed_peers.is_empty()); + // What such a request delivers is here all the same. + late.reply = delivering(&[1, 2]); assert_eq!( - settle_column_answer(&mut pending, peer, &[1], &[1]), - Some(ColumnAttemptOutcome::Retry { + settle_column_report(&mut pending, &late), + ColumnAttemptOutcome::Complete + ); + } + + #[test] + fn a_sidecar_the_checks_refuse_stays_missing_and_is_held_against_its_sender() { + // The right root and index used to be enough to take a column off the + // lookup and clear its sender, whatever its cells and proofs were. + let custodian = PeerId::random(); + let mut pending = column_lookup(vec![1, 2], &[(1, custodian), (2, custodian)]); + + let verdicts = vec![(1, ColumnVerdict::Delivered), (2, ColumnVerdict::Rejected)]; + let answer = column_report(custodian, &[1, 2], ColumnReply::Answered(verdicts)); + assert_eq!( + settle_column_report(&mut pending, &answer), + ColumnAttemptOutcome::Retry { attempts: 1, - backoff: Duration::from_millis(INITIAL_BACKOFF_MS), - }) + backoff: FIRST_BACKOFF, + } ); - assert_eq!(pending.columns, vec![4]); + assert_eq!(pending.columns, vec![2]); + assert!(pending.failed_peers.contains(&custodian)); + } + + #[test] + fn an_undecided_sidecar_stays_missing_without_blaming_its_sender() { + let peer = PeerId::random(); + let mut pending = column_lookup(vec![1], &[(1, peer)]); + + let verdicts = vec![(1, ColumnVerdict::Undecided)]; + let answer = column_report(peer, &[1], ColumnReply::Answered(verdicts)); + assert!(matches!( + settle_column_report(&mut pending, &answer), + ColumnAttemptOutcome::Retry { .. } + )); + assert_eq!(pending.columns, vec![1]); assert!(pending.failed_peers.is_empty()); } + /// A beacon server with `peers` connected, each with the custody it has + /// told this node (`None`: nothing yet). + async fn beacon_server_with(peers: &[(PeerId, Option>)]) -> P2PServer { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + for (peer, custody) in peers { + server + .connected_peers + .insert(*peer, ConnectionDirection::Inbound); + if let Some(custody) = custody { + server.peer_custody.insert(*peer, custody.clone()); + } + } + server + } + + /// A shaped fulu sidecar of `index`, with the root of the block it names. + fn sidecar_of(index: u64) -> (DataColumnSidecar, H256) { + let sidecar = DataColumnSidecar::Fulu(crate::test_support::valid_shaped_sidecar(5, index)); + let root = sidecar.block_root(); + (sidecar, root) + } + + #[tokio::test] + async fn a_column_no_known_custodian_covers_goes_only_to_a_peer_that_has_said_nothing() { + // Uniformly random used to include peers whose recorded custody + // excludes the column, and asking one of them is a round spent on a + // request that cannot be answered. + let holder = PeerId::random(); + let elsewhere = PeerId::random(); + let silent = PeerId::random(); + let mut server = beacon_server_with(&[ + (holder, Some(vec![1])), + (elsewhere, Some(vec![9])), + (silent, None), + ]) + .await; + let root = H256::repeat_byte(0x51); + + assert!(fetch_data_columns_from_peer(&mut server, root, vec![1, 2, 3]).await); + + let pending = &server.pending_column_requests[&root]; + assert_eq!( + pending.asked, + HashMap::from([(1, holder), (2, silent), (3, silent)]) + ); + assert_eq!(server.outbound_requests.len(), 2); + assert!(server.outbound_requests.values().all(|kind| matches!( + kind, + PendingRequestKind::Columns { lookup, .. } if *lookup == pending.lookup + ))); + } + + #[tokio::test] + async fn a_column_nobody_is_known_to_keep_is_asked_beyond_custody_once_per_lookup() { + // A held block's column that no connected peer keeps used to cost + // every remaining rung of the ladder, a random peer a rung, then an + // error and the max-retries count, once a slot. With nobody of unknown + // custody connected, one peer that has said what it keeps is asked + // once (it may hold more), and then the lookup ends quietly. + let elsewhere = PeerId::random(); + let mut server = beacon_server_with(&[(elsewhere, Some(vec![9]))]).await; + let root = H256::repeat_byte(0x52); + + assert!(fetch_data_columns_from_peer(&mut server, root, vec![2]).await); + let pending = &server.pending_column_requests[&root]; + let lookup = pending.lookup; + assert_eq!(pending.asked, HashMap::from([(2, elsewhere)])); + assert!(pending.asked_beyond_custody); + + // It does not have 2 after all. + let mut empty = column_report(elsewhere, &[2], delivering(&[])); + empty.request.block_root = root; + empty.request.lookup = lookup; + assert!(matches!( + report_column_request(&mut server, &empty), + Some(ColumnAttemptOutcome::Retry { .. }) + )); + + let before = metrics::data_column_fetch_failures_total("no_custodian"); + retry_column_lookup(&mut server, root, lookup).await; + + assert!(server.pending_column_requests.is_empty()); + assert_eq!( + server.outbound_requests.len(), + 1, + "nobody was asked a second time" + ); + assert!( + metrics::data_column_fetch_failures_total("no_custodian") > before, + "counted as having nobody to ask, not as running out of retries" + ); + } + + #[tokio::test] + async fn at_the_tip_a_peer_that_answered_short_is_asked_again() { + // The devnet's shape: a custodian of one column, and the only peer of + // unknown custody asked for the rest, which at the tip has 2 but not + // yet 3. Writing it off would leave 3 nobody to ask until the next + // slot's re-ask, the ~12 s hold this lookup exists to remove. + let custodian = PeerId::random(); + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(custodian, Some(vec![1])), (silent, None)]).await; + let root = H256::repeat_byte(0x56); + let now = server.wire.beacon().expect("a beacon wire").wall_slot(); + + assert!(fetch_data_columns_from_peer(&mut server, root, vec![1, 2, 3]).await); + let pending = server + .pending_column_requests + .get_mut(&root) + .expect("a lookup"); + pending.slot = Some(now); + let lookup = pending.lookup; + let mut short = column_report(silent, &[2, 3], delivering(&[2])); + short.request.block_root = root; + short.request.lookup = lookup; + assert!(matches!( + report_column_request(&mut server, &short), + Some(ColumnAttemptOutcome::Retry { .. }) + )); + let mut whole = column_report(custodian, &[1], delivering(&[1])); + whole.request.block_root = root; + whole.request.lookup = lookup; + report_column_request(&mut server, &whole); + + retry_column_lookup(&mut server, root, lookup).await; + + assert_eq!( + server.pending_column_requests[&root].asked, + HashMap::from([(3, silent)]) + ); + } + + #[tokio::test] + async fn past_the_tip_each_peer_is_asked_for_a_column_once() { + // The mainnet case: a held block re-asked a slot later for a + // column no connected peer is known to keep. Nothing is still arriving + // by then, so a peer that answered without it is not asked again: each + // peer of unknown custody is asked once, and then the lookup ends + // quietly instead of walking the ladder. + let [first, second] = [PeerId::random(), PeerId::random()]; + let mut server = beacon_server_with(&[(first, None), (second, None)]).await; + let root = H256::repeat_byte(0x58); + + assert!(fetch_data_columns_from_peer(&mut server, root, vec![2]).await); + let pending = server + .pending_column_requests + .get_mut(&root) + .expect("a lookup"); + pending.slot = Some(5); + let lookup = pending.lookup; + let before = metrics::data_column_fetch_failures_total("no_custodian"); + + for _ in 0..2 { + let asked = server.pending_column_requests[&root].asked[&2]; + let mut empty = column_report(asked, &[2], delivering(&[])); + empty.request.block_root = root; + empty.request.lookup = lookup; + report_column_request(&mut server, &empty); + retry_column_lookup(&mut server, root, lookup).await; + } + + assert!(server.pending_column_requests.is_empty()); + assert_eq!(server.outbound_requests.len(), 2, "one ask each"); + assert!(metrics::data_column_fetch_failures_total("no_custodian") > before); + } + + #[tokio::test] + async fn a_retry_asks_only_for_what_the_store_does_not_have() { + // Gossip often delivers a column while a lookup started at the tip is + // still asking for it. + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(silent, None)]).await; + let (stored, root) = sidecar_of(1); + server.store.put_data_column(&stored).expect("stored"); + let mut pending = column_lookup(vec![1, 2], &[]); + pending.slot = Some(5); + pending.retry_due = true; + server.pending_column_requests.insert(root, pending); + + retry_column_lookup(&mut server, root, LOOKUP).await; + + let pending = &server.pending_column_requests[&root]; + assert_eq!(pending.columns, vec![2]); + assert_eq!(pending.asked, HashMap::from([(2, silent)])); + assert!(!pending.retry_due); + + // Once the store has everything, the lookup is complete and the next + // retry asks nobody. + let (also_stored, _) = sidecar_of(2); + server.store.put_data_column(&also_stored).expect("stored"); + let asked_before = server.outbound_requests.len(); + retry_column_lookup(&mut server, root, LOOKUP).await; + assert!(server.pending_column_requests.is_empty()); + assert_eq!(server.outbound_requests.len(), asked_before); + } + + #[tokio::test] + async fn a_retry_for_a_lookup_that_was_replaced_does_nothing() { + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(silent, None)]).await; + let root = H256::repeat_byte(0x53); + let mut pending = column_lookup(vec![1], &[]); + pending.retry_due = true; + server.pending_column_requests.insert(root, pending); + + retry_column_lookup(&mut server, root, LOOKUP + 1).await; + + assert!(server.outbound_requests.is_empty()); + assert!(server.pending_column_requests[&root].retry_due); + } + + #[tokio::test] + async fn a_column_merged_into_a_lookup_is_asked_at_once() { + // The re-ask that merges it used to send nothing while the lookup was + // younger than `STALE_COLUMN_LOOKUP`, leaving it to the lookup's next + // retry, which a hung request could hold back for its whole timeout. + let holder = PeerId::random(); + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(holder, Some(vec![1])), (silent, None)]).await; + let root = H256::repeat_byte(0x54); + server + .pending_column_requests + .insert(root, column_lookup(vec![1, 2], &[(1, holder)])); + + ask_unasked_columns(&mut server, root).await; + + assert_eq!( + server.pending_column_requests[&root].asked, + HashMap::from([(1, holder), (2, silent)]) + ); + let sent: Vec<_> = server.outbound_requests.values().collect(); + assert!(matches!( + sent.as_slice(), + [PendingRequestKind::Columns { columns, lookup: LOOKUP, .. }] if *columns == vec![2] + )); + } + + #[tokio::test] + async fn an_answer_settles_its_lookup_once_its_sidecars_are_checked() { + use crate::beacon::column_checks::tests::RecordingChain; + + let peer = PeerId::random(); + let mut server = beacon_server_with(&[(peer, None)]).await; + let (sender, _forwarded) = tokio::sync::mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(RecordingChain(sender))); + // Stored already, so the checks find the column here without the + // parent state a sidecar of a new block would need. + let (sidecar, root) = sidecar_of(1); + server.store.put_data_column(&sidecar).expect("stored"); + server + .pending_column_requests + .insert(root, column_lookup(vec![1], &[(1, peer)])); + + let (reports, mut checked) = tokio::sync::mpsc::unbounded_channel(); + let request = column_request(root, peer, &[1]); + let outcome = take_column_answer(&mut server, request, vec![sidecar], move |report| { + let _ = reports.send(report); + }); + + assert_eq!(outcome, None, "nothing is settled before the checks"); + assert_eq!( + server.pending_column_requests[&root].asked, + HashMap::from([(1, peer)]), + "a column being checked is not asked of anyone else" + ); + + let report = checked.recv().await.expect("the checks report back"); + assert_eq!( + report.reply, + ColumnReply::Answered(vec![(1, ColumnVerdict::Delivered)]) + ); + assert_eq!( + report_column_request(&mut server, &report), + Some(ColumnAttemptOutcome::Complete) + ); + assert!( + server.pending_column_requests.is_empty(), + "a complete lookup is retired" + ); + } + + #[tokio::test] + async fn an_answer_no_lookup_waits_on_is_still_checked_and_forwarded() { + use crate::beacon::column_checks::tests::{Forwarded, RecordingChain}; + + let peer = PeerId::random(); + let mut server = beacon_server_with(&[(peer, None)]).await; + let (sender, mut forwarded) = tokio::sync::mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(RecordingChain(sender))); + // Its parent is not in the store, so the chain actor parks it. + let (sidecar, root) = sidecar_of(1); + + let (reports, mut checked) = tokio::sync::mpsc::unbounded_channel(); + let request = column_request(root, peer, &[1]); + let outcome = + take_column_answer(&mut server, request, vec![sidecar.clone()], move |report| { + let _ = reports.send(report); + }); + + assert_eq!(outcome, None); + assert_eq!( + forwarded.recv().await, + Some(Forwarded::AwaitingParent(vec![sidecar])) + ); + let report = checked.recv().await.expect("the checks report back"); + assert_eq!(report_column_request(&mut server, &report), None); + assert!( + server.pending_column_requests.is_empty(), + "nothing is resurrected" + ); + } + + #[tokio::test] + async fn an_answer_at_the_end_of_the_ladder_gives_the_lookup_up() { + // A custodian, so running out is a failure to fetch what someone + // should have had. + let peer = PeerId::random(); + let mut server = beacon_server_with(&[(peer, Some(vec![1, 2]))]).await; + let root = H256::repeat_byte(0x55); + let mut pending = column_lookup(vec![1, 2], &[(1, peer), (2, peer)]); + pending.attempts = MAX_FETCH_RETRIES; + server.pending_column_requests.insert(root, pending); + let before = metrics::data_column_fetch_failures_total("max_retries"); + + let request = column_request(root, peer, &[1, 2]); + let outcome = take_column_answer(&mut server, request, Vec::new(), |_| { + unreachable!("an empty answer has nothing to check") + }); + + assert_eq!( + outcome, + Some(ColumnAttemptOutcome::GiveUp { + attempts: MAX_FETCH_RETRIES + }) + ); + assert!(server.pending_column_requests.is_empty()); + assert!(metrics::data_column_fetch_failures_total("max_retries") > before); + } + + #[tokio::test] + async fn a_ladder_spent_on_columns_nobody_keeps_ends_as_no_custodian() { + // Not an error: nobody connected was known to hold them. + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(silent, None)]).await; + let root = H256::repeat_byte(0x57); + let mut pending = column_lookup(vec![1], &[(1, silent)]); + pending.attempts = MAX_FETCH_RETRIES; + server.pending_column_requests.insert(root, pending); + let before = metrics::data_column_fetch_failures_total("no_custodian"); + + let request = column_request(root, silent, &[1]); + let outcome = take_column_answer(&mut server, request, Vec::new(), |_| { + unreachable!("an empty answer has nothing to check") + }); + + assert!(matches!(outcome, Some(ColumnAttemptOutcome::GiveUp { .. }))); + assert!(server.pending_column_requests.is_empty()); + assert!(metrics::data_column_fetch_failures_total("no_custodian") > before); + } + #[test] fn an_out_of_range_custody_group_count_records_nothing() { // A count outside CUSTODY_REQUIREMENT..=NUMBER_OF_CUSTODY_GROUPS diff --git a/docs/metrics.md b/docs/metrics.md index c9264b677..b657b5f88 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -600,7 +600,7 @@ spec. | `lean_data_columns_stored_total` | Counter | Data column sidecars verified and written to `Table::DataColumns` | On the chain actor storing a sidecar the p2p layer verified | | | | `lean_data_columns_rejected_total` | Counter | Sidecars the p2p layer's chain checks dropped, by reason | On each drop in `beacon::column_checks`, except an already stored sidecar | reason=malformed,future_slot,finalized,not_after_parent,unknown_proposer,bad_signature,wrong_proposer,finalized_not_ancestor,parent_not_ready,inclusion_proof,kzg,internal | | | `lean_data_column_kzg_verify_seconds` | Histogram | Time spent batch-verifying one sidecar's cells against its own commitments | On each KZG batch verification, in gossip validation or the chain checks | | 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0 | -| `lean_data_column_fetch_failures_total` | Counter | `DataColumnsByRoot` lookups this node gave up on, by reason | On lookup abandonment | reason=no_peers,max_retries | | +| `lean_data_column_fetch_failures_total` | Counter | `DataColumnsByRoot` lookups this node gave up on, by reason | On lookup abandonment | reason=no_peers,max_retries,no_custodian | | | `lean_blocks_held_for_columns` | Gauge | Blocks held out of fork choice pending their custody columns | On every hold, release, and finality eviction of the held-block set | | | | `lean_sidecars_awaiting_parent` | Gauge | Sidecars parked until their block's parent has a post-state | On every park, replay, and finality eviction of the parked set | | | | `lean_envelopes_awaiting_block` | Gauge | Gloas execution payload envelopes held until their block is imported | On every hold, release, and finality eviction of the held envelopes | | | @@ -622,9 +622,12 @@ of a hold: fetch or gossip should complete well inside the retry ladder `lean_data_column_fetch_failures_total` counts down. A gauge that sits above zero for several slots means either the fetch is failing — check whether `lean_data_column_fetch_failures_total`'s `no_peers` or `max_retries` reason -is climbing — or no reachable peer actually serves the missing columns; short -of that, only finality passing the held block's slot clears it, which can be -minutes away. +is climbing — or no reachable peer actually serves the missing columns, which +the `no_custodian` reason counts: a lookup that ends without finding anyone +connected holding the missing columns, because none is known to custody them +or every peer asked after the block's slot answered without them. +Short of that, only finality passing the held block's slot clears it, which +can be minutes away. **Read `lean_sidecars_awaiting_parent` beside it.** A sidecar whose block's parent has no post-state yet is parked rather than dropped, and a held block is From c03ef90c3daa2db94e4f6022f9788cd0df468c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:50:35 -0300 Subject: [PATCH 3/7] docs(beacon_wire): describe the by-root column lookup as it now runs Each request settles on its own after the checks, a peer that answers short is asked again only while the block is at the tip, the store is read before asking again, and the columns no known custodian covers go to a peer of unknown custody, or once per lookup to one of known custody when none is connected. --- docs/beacon_wire.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 2cef98120..f8d8d2357 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -720,11 +720,19 @@ hits this on every held block, and without it such a follower imported roughly one block per slot, waiting out the redrive below for a request gossip was never going to answer. Either way, `redrive_held_blocks` runs on every slot tick: for each block still held it sends the same request for -exactly the columns still missing, with -`MAX_FETCH_RETRIES` attempts and backoff doubling from `INITIAL_BACKOFF_MS`, a -peer that has already failed this lookup excluded until the whole pool is -exhausted. A lookup that runs out of peers or -retries stops asking until the next tick asks again; see +exactly the columns still missing. One lookup is split across peers, and each +request settles on its own once its answer has been through the chain checks: +what passed leaves the lookup, and what the peer left out is asked again at +once, without waiting for the lookup's other requests, with +`MAX_FETCH_RETRIES` attempts and backoff doubling from `INITIAL_BACKOFF_MS`. +Only a request that left something out spends an attempt. A peer whose request +failed, or that sent a sidecar the checks refused, is not asked again in that +lookup. A peer that answered without a column is asked for it again only +while the block is at the tip (it may not have received the column yet), and +then after every other candidate; past the tip it is not. Before a +retry asks, the columns the store already holds (gossip often delivers them +meanwhile) are taken off. A lookup that runs out of retries, or has nobody +left to ask, stops asking until the next tick asks again; see `lean_data_column_fetch_failures_total` in [metrics.md](./metrics.md). The block is released the moment its last missing column arrives, and dropped along with any pending descendants once finality passes its slot, whichever @@ -739,8 +747,11 @@ the spec's own observation that "due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to". The count comes from `metadata/3`, requested once per connection right behind `Status`, and is seeded from the ENR `cgc` for a peer this node dialed; a peer that has supplied -neither is not assumed to custody anything, and the by-root path falls back to -asking one at random for whatever no known custodian covers. At mainnet's +neither is not assumed to custody anything, and both paths send whatever no +known custodian covers to such a peer rather than to one that has said what it +keeps. The by-root path asks one at a time; only when none is connected does +it ask a peer of known custody for those columns, once per lookup, since such +a peer may hold more than it custodies. At mainnet's `CUSTODY_REQUIREMENT` a peer holds 8 of 128 columns, so this is the difference between a request that can be answered and one that usually cannot. From fd24089c863c0170028297be08da510c5afe1120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:08:51 -0300 Subject: [PATCH 4/7] fix(p2p): report column verdicts from a drop guard and take back an undelivered keep The completion callback ran from the end of the check task, so a panic outside the blocking checks or a task ended early left the lookup with its columns on the wire until it went stale. Verdicts now report when dropped, with anything not judged as undecided. A kept sidecar counted as delivered even when handing it to the chain actor failed, so the lookup could retire as complete for a column the chain never got. Those are now reported undecided, and asked again. --- crates/net/p2p/src/beacon/column_checks.rs | 178 ++++++++++++++++----- 1 file changed, 138 insertions(+), 40 deletions(-) diff --git a/crates/net/p2p/src/beacon/column_checks.rs b/crates/net/p2p/src/beacon/column_checks.rs index 43cabd912..c5098fe14 100644 --- a/crates/net/p2p/src/beacon/column_checks.rs +++ b/crates/net/p2p/src/beacon/column_checks.rs @@ -31,13 +31,14 @@ use crate::{P2PServer, metrics}; /// sent it is to blame if not. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ColumnVerdict { - /// Passed and went to the chain actor, or was already stored: the column - /// is here. + /// Passed and was handed to the chain actor, or was already stored: the + /// column is here. Delivered, /// Not judged, through no fault of the sender: parked until its block can /// judge it, dropped for a reason that is not the sender's (an `Ignore`), - /// or never checked, because this node had no chain actor to check it for - /// or its check panicked. + /// or never checked or never handed over, because this node had no chain + /// actor to check it for or reach, its check panicked, or its task ended + /// first. Undecided, /// Refused as invalid: the sender sent something it should not have. Rejected, @@ -55,9 +56,62 @@ impl From<&ChainVerdict> for ColumnVerdict { } /// A completion callback: each sidecar's column index with its -/// [`ColumnVerdict`]. +/// [`ColumnVerdict`], in no particular order. pub(crate) type Done = Box) + Send>; +/// The verdicts of one batch, handed to its [`Done`] when dropped: those +/// recorded, and every other sidecar of the batch as +/// [`ColumnVerdict::Undecided`]. +/// +/// Reported on drop rather than by a call at the end of the checks, so every +/// way out reports, a panic or a task ended early included: a lookup that +/// never hears back keeps its columns on the wire until it goes stale. +struct Verdicts { + done: Option, + /// Column indices of the batch with no verdict recorded yet. + unjudged: Vec, + recorded: Vec<(u64, ColumnVerdict)>, +} + +impl Verdicts { + fn new(done: Option, sidecars: &[DataColumnSidecar]) -> Self { + let unjudged = sidecars.iter().map(DataColumnSidecar::index).collect(); + Self { + done, + unjudged, + recorded: Vec::new(), + } + } + + fn record(&mut self, index: u64, verdict: ColumnVerdict) { + if let Some(position) = self.unjudged.iter().position(|&left| left == index) { + self.unjudged.swap_remove(position); + } + self.recorded.push((index, verdict)); + } + + /// Take back `indices`' delivery: the chain actor never got them. + fn not_delivered(&mut self, indices: &[u64]) { + for (index, verdict) in &mut self.recorded { + if *verdict == ColumnVerdict::Delivered && indices.contains(index) { + *verdict = ColumnVerdict::Undecided; + } + } + } +} + +impl Drop for Verdicts { + fn drop(&mut self) { + let Some(done) = self.done.take() else { + return; + }; + let mut verdicts = std::mem::take(&mut self.recorded); + let unjudged = self.unjudged.drain(..); + verdicts.extend(unjudged.map(|index| (index, ColumnVerdict::Undecided))); + done(verdicts); + } +} + /// Run the chain checks on `sidecars`, then send the chain actor the ones that /// passed as one batch and the ones waiting on a block as another. /// @@ -74,53 +128,41 @@ pub(crate) fn check_and_forward(server: &P2PServer, sidecars: Vec, done: Option, ) { - let run_done = |done: Option, verdicts: Vec<(u64, ColumnVerdict)>| { - if let Some(done) = done { - done(verdicts); - } - }; - let blockchain = match server.blockchain.clone() { - Some(blockchain) if !sidecars.is_empty() => blockchain, - _ => { - run_done(done, undecided(&sidecars)); - return; - } + let mut verdicts = Verdicts::new(done, &sidecars); + let Some(blockchain) = server.blockchain.clone().filter(|_| !sidecars.is_empty()) else { + return; }; let store = server.store.clone(); let permits = server.column_check_permits.clone(); tokio::spawn(async move { let mut checks = Vec::with_capacity(sidecars.len()); - let mut verdicts = Vec::with_capacity(sidecars.len()); - let mut sidecars = sidecars.into_iter(); - while let Some(sidecar) = sidecars.next() { + for sidecar in sidecars { // The semaphore is never closed, so this only returns once a // permit is free. let Ok(permit) = permits.clone().acquire_owned().await else { - verdicts.push((sidecar.index(), ColumnVerdict::Undecided)); - verdicts.extend(undecided(sidecars.as_slice())); break; }; let store = store.clone(); - let index = sidecar.index(); let check = tokio::task::spawn_blocking(move || { let _permit = permit; let verdict = column::chain_checks_for(&store, &sidecar, unix_now_ms()); (sidecar, verdict) }); - checks.push((index, check)); + checks.push(check); } let mut keep = Vec::new(); + let mut kept = Vec::new(); let mut awaiting_parent = Vec::new(); - for (index, check) in checks { + for check in checks { // A panic inside the checks (a `Store` read's `.expect()` on a DB // error, say) costs that one sidecar, which is what dropping it // would have done anyway. Tokio has already caught it. @@ -128,13 +170,15 @@ pub(crate) fn check_and_forward_then( Ok(checked) => checked, Err(err) => { error!(%err, "A data column check panicked; dropping the sidecar"); - verdicts.push((index, ColumnVerdict::Undecided)); continue; } }; - verdicts.push((index, ColumnVerdict::from(&verdict))); + verdicts.record(sidecar.index(), ColumnVerdict::from(&verdict)); match verdict { - ChainVerdict::Keep => keep.push(sidecar), + ChainVerdict::Keep => { + kept.push(sidecar.index()); + keep.push(sidecar); + } ChainVerdict::AwaitParent => awaiting_parent.push(sidecar), ChainVerdict::Drop(outcome) => count_drop(outcome), } @@ -143,7 +187,10 @@ pub(crate) fn check_and_forward_then( if !keep.is_empty() { let _ = blockchain .new_data_column_sidecars(keep) - .inspect_err(|err| warn!(%err, "Failed to forward checked data column sidecars")); + .inspect_err(|err| { + warn!(%err, "Failed to forward checked data column sidecars"); + verdicts.not_delivered(&kept); + }); } if !awaiting_parent.is_empty() { let _ = blockchain @@ -152,18 +199,10 @@ pub(crate) fn check_and_forward_then( |err| warn!(%err, "Failed to forward data column sidecars awaiting a parent"), ); } - run_done(done, verdicts); + // `verdicts` reports as it drops here. }); } -/// Every sidecar of `sidecars` as [`ColumnVerdict::Undecided`]. -fn undecided(sidecars: &[DataColumnSidecar]) -> Vec<(u64, ColumnVerdict)> { - sidecars - .iter() - .map(|sidecar| (sidecar.index(), ColumnVerdict::Undecided)) - .collect() -} - /// Count a dropped sidecar under its reason. /// /// An already stored one is not counted: consecutive range batches ask for @@ -384,4 +423,63 @@ pub(crate) mod tests { Some(vec![(3, ColumnVerdict::Undecided)]) ); } + + /// Verdicts reported through `done`, for the guard tests below. + fn reporting() -> ( + Option, + mpsc::UnboundedReceiver>, + ) { + let (sender, received) = mpsc::unbounded_channel(); + let done: Done = Box::new(move |verdicts| { + let _ = sender.send(verdicts); + }); + (Some(done), received) + } + + /// A task that ends before judging every sidecar (a panic outside the + /// blocking checks, or a runtime shutting down) still settles the lookup, + /// with what it did not judge as undecided. + #[test] + fn verdicts_dropped_early_report_the_rest_as_undecided() { + let (done, mut reported) = reporting(); + let batch = [ + DataColumnSidecar::Fulu(valid_shaped_sidecar(5, 3)), + DataColumnSidecar::Fulu(valid_shaped_sidecar(5, 4)), + ]; + let mut verdicts = Verdicts::new(done, &batch); + verdicts.record(3, ColumnVerdict::Delivered); + drop(verdicts); + + assert_eq!( + reported.try_recv().ok(), + Some(vec![ + (3, ColumnVerdict::Delivered), + (4, ColumnVerdict::Undecided) + ]) + ); + } + + /// A sidecar that passed but never reached the chain actor (its mailbox + /// closed) is not delivered: the lookup asks for it again. + #[test] + fn a_kept_sidecar_the_chain_never_got_is_not_delivered() { + let (done, mut reported) = reporting(); + let batch = [ + DataColumnSidecar::Fulu(valid_shaped_sidecar(5, 5)), + DataColumnSidecar::Fulu(valid_shaped_sidecar(5, 6)), + ]; + let mut verdicts = Verdicts::new(done, &batch); + verdicts.record(5, ColumnVerdict::Delivered); + verdicts.record(6, ColumnVerdict::Delivered); + verdicts.not_delivered(&[5]); + drop(verdicts); + + assert_eq!( + reported.try_recv().ok(), + Some(vec![ + (5, ColumnVerdict::Undecided), + (6, ColumnVerdict::Delivered) + ]) + ); + } } From 2c51d42ca24d6f92a82c70be8d624306b4d2397b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:17:42 -0300 Subject: [PATCH 5/7] fix(p2p): ask a peer for the uncovered columns it has not missed Past the tip, a peer of unknown custody that had answered without one uncovered column was passed over for all of them, so a column merged in after its short answer went to nobody and waited for the next slot's re-ask. Uncovered columns are now assigned per column: a peer takes the ones it may still be asked for, and another peer the rest. The once-per- lookup ask beyond custody follows the same rule. Also from the third review round: - A replaced lookup's report that carried an invalid sidecar writes its sender off for the new lookup too; a failure from it still does not. - The plan's `excluded` counts only connected peers. - `settle_column_report` documents that a short report riding on a zero-backoff retry spends nothing, and why that is bounded. - The unreachable `ColumnIndices` error says so, and a missing blank line. - Tests: the per-column assignment past the tip, the invalid stale report, and a lookup reading the tip from its held block in the store. --- crates/net/p2p/src/req_resp/handlers.rs | 232 ++++++++++++++++++------ 1 file changed, 177 insertions(+), 55 deletions(-) diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 0117c512a..9edcfae76 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -1176,9 +1176,10 @@ pub async fn fetch_data_columns_from_peer( let column_indices = match ColumnIndices::try_from(peer_columns.clone()) { Ok(indices) => indices, Err(err) => { - // A caller asking for more than one block's worth of columns is a - // programming error on the chain-actor side, not a peer or wire - // fault, so this does not retry. + // Unreachable: a lookup holds distinct column indices, so no + // request can name more than NUMBER_OF_COLUMNS of them. Were it + // ever reached, these columns would stay unasked, and the lookup + // would end once nothing else of it is on the wire. error!(%block_root, ?err, "Too many columns requested in one DataColumnsByRoot lookup"); continue; } @@ -1243,7 +1244,7 @@ struct ColumnPlan { /// The lookup's id: the existing one's, or a fresh one's. lookup: u64, by_peer: HashMap>, - /// How many peers the lookup's history rules out. + /// How many connected peers the lookup's history rules out. excluded: usize, /// The peer asked for columns outside its known custody, if one is. beyond_custody: Option, @@ -1253,22 +1254,24 @@ struct ColumnPlan { /// there is none yet). /// /// Each column goes to a peer known to custody it. Whatever no known -/// custodian covers goes to one peer whose custody is unknown, the same bet -/// the range path makes (see [`peers_of_unknown_custody`]). +/// custodian covers goes to a peer whose custody is unknown, the same bet the +/// range path makes (see [`peers_of_unknown_custody`]), one peer taking as many +/// of those columns as it may be asked for, so they cost as few requests as +/// possible. /// /// A peer that already answered this lookup without a column is asked for it /// again only while the block is at the tip (its slot is the wall clock's or /// later), and then after every peer that has not: at the tip it may have /// received the column since, as `hold_block_for_columns` explains. Past the /// tip there is no gossip left to bring it, so asking again would only spend -/// the ladder. A lookup that has not read the block's slot yet counts as past -/// the tip. +/// the ladder; that peer is still asked for the columns it has not missed. A +/// lookup that has not read the block's slot yet counts as past the tip. /// -/// Only when no peer of unknown custody is left do uncovered columns go to a -/// peer that has said what it keeps, and only once per lookup. Such a peer -/// may hold more than it custodies, and asking one at random each slot is all -/// this node did before custody was known; but walking every one of them, a -/// round each, is what a lookup for a column nobody holds used to cost. +/// Only when no peer of unknown custody can take an uncovered column does it +/// go to a peer that has said what it keeps, and only once per lookup. Such a +/// peer may hold more than it custodies, and asking one at random each slot is +/// all this node did before custody was known; but walking every one of them, +/// a round each, is what a lookup for a column nobody holds used to cost. fn plan_column_requests(server: &P2PServer, block_root: H256, columns: &[u64]) -> ColumnPlan { let no_peers = HashSet::new(); let no_misses = HashSet::new(); @@ -1285,19 +1288,20 @@ fn plan_column_requests(server: &P2PServer, block_root: H256, columns: &[u64]) - let (mut by_peer, uncovered) = group_columns_by_custody_peer(server, columns, failed, missed, at_tip); + let excluded = failed + .iter() + .filter(|peer| server.connected_peers.contains_key(*peer)) + .count(); let mut plan = ColumnPlan { lookup, by_peer: HashMap::new(), - excluded: failed.len(), + excluded, beyond_custody: None, }; - if uncovered.is_empty() { - plan.by_peer = by_peer; - return plan; - } - let misses = |peer: &PeerId| { - uncovered + let may_ask = |peer: &PeerId, column: u64| at_tip || !missed.contains(&(*peer, column)); + let misses = |peer: &PeerId, columns: &[u64]| { + columns .iter() .filter(|&&column| missed.contains(&(*peer, column))) .count() @@ -1305,42 +1309,74 @@ fn plan_column_requests(server: &P2PServer, block_root: H256, columns: &[u64]) - let unknown: Vec = peers_of_unknown_custody(server, usize::MAX) .into_iter() .filter(|peer| !failed.contains(peer)) - .filter(|peer| at_tip || misses(peer) == 0) .collect(); - let fewest_misses = unknown.iter().map(misses).min(); - let candidates: Vec = match fewest_misses { - Some(fewest) => unknown + let mut remaining = uncovered; + let mut picked: HashSet = HashSet::new(); + while !remaining.is_empty() { + let candidates: Vec = unknown + .iter() + .filter(|peer| !picked.contains(*peer)) + .filter(|peer| remaining.iter().any(|&column| may_ask(peer, column))) + .copied() + .collect(); + let Some(fewest) = candidates.iter().map(|peer| misses(peer, &remaining)).min() else { + break; + }; + let least_missed: Vec = candidates .into_iter() - .filter(|peer| misses(peer) == fewest) - .collect(), - None if !beyond_custody_spent => server + .filter(|peer| misses(peer, &remaining) == fewest) + .collect(); + let Some(&peer) = least_missed.choose(&mut rand::thread_rng()) else { + break; + }; + let (asked, left): (Vec, Vec) = remaining + .iter() + .partition(|&&column| may_ask(&peer, column)); + debug!( + %block_root, + %peer, + count = asked.len(), + "No connected peer is known to custody these columns; asking one whose custody is unknown" + ); + picked.insert(peer); + by_peer.entry(peer).or_default().extend(asked); + remaining = left; + } + + if !remaining.is_empty() && !beyond_custody_spent { + let known: Vec = server .connected_peers .keys() - .filter(|peer| !failed.contains(*peer) && misses(peer) == 0) + .filter(|peer| server.peer_custody.contains_key(*peer) && !failed.contains(*peer)) + .filter(|peer| { + remaining + .iter() + .any(|&column| !missed.contains(&(**peer, column))) + }) .copied() - .collect(), - None => Vec::new(), - }; - match candidates.choose(&mut rand::thread_rng()) { - Some(&peer) => { - let custody_known = server.peer_custody.contains_key(&peer); + .collect(); + if let Some(&peer) = known.choose(&mut rand::thread_rng()) { + let (asked, left): (Vec, Vec) = remaining + .iter() + .partition(|&&column| !missed.contains(&(peer, column))); debug!( %block_root, %peer, - custody_known, - count = uncovered.len(), - "No connected peer is known to custody these columns; asking one that might hold them" + count = asked.len(), + "No peer of unknown custody can take these columns; asking one beyond its custody, once" ); - if custody_known { - plan.beyond_custody = Some(peer); - } - by_peer.entry(peer).or_default().extend(uncovered); + plan.beyond_custody = Some(peer); + by_peer.entry(peer).or_default().extend(asked); + remaining = left; } - None => debug!( + } + + if !remaining.is_empty() { + debug!( %block_root, - count = uncovered.len(), + count = remaining.len(), "No connected peer is known to custody these columns, and none is left to ask" - ), + ); } plan.by_peer = by_peer; plan @@ -1716,15 +1752,19 @@ impl ColumnReport { /// the tip a peer often has not received the column yet, which is why the /// chain actor waits a slot before asking for a block still at the tip /// (see `hold_block_for_columns`). Such a peer is recorded in `missed` - /// instead, and asked for that column again only after every peer that - /// has not missed it. + /// instead (see `plan_column_requests` for when it is asked again). fn peer_at_fault(&self) -> bool { - match &self.reply { - ColumnReply::Failed => true, - ColumnReply::Answered(verdicts) => verdicts - .iter() - .any(|(_, verdict)| *verdict == ColumnVerdict::Rejected), - } + matches!(self.reply, ColumnReply::Failed) || self.sent_invalid() + } + + /// Whether the answer carried a sidecar the checks refused. + fn sent_invalid(&self) -> bool { + let ColumnReply::Answered(verdicts) = &self.reply else { + return false; + }; + verdicts + .iter() + .any(|(_, verdict)| *verdict == ColumnVerdict::Rejected) } } @@ -1765,6 +1805,13 @@ enum ColumnAttemptOutcome { /// lookup after it was sent, which is asked for at once. However many of a /// lookup's requests come back short, one retry is due at a time, and it asks /// for all of their leftovers. +/// +/// A short request whose leftover rides on a retry already due spends nothing +/// either, even when that retry is the zero-backoff one a whole share +/// scheduled: one leftover can then be asked again without backing off. That +/// is bounded: it takes a whole share to schedule such a retry, a whole share +/// takes columns off the lookup, and the next request to come back short +/// after the retry has fired spends an attempt as usual. fn settle_column_report( pending: &mut PendingColumnRequest, report: &ColumnReport, @@ -1774,9 +1821,14 @@ fn settle_column_report( pending.columns.retain(|column| !delivered.contains(column)); // A request of a lookup this one replaced: its columns are here all the - // same, but it is not one of this lookup's requests, so it settles nothing - // else. + // same, but it is not one of this lookup's requests, so it closes none of + // them and spends nothing. A peer that sent an invalid sidecar for this + // root is still not one to ask for it again; a failure, though, may be + // what got the old lookup replaced, and says little about the peer now. if request.lookup != pending.lookup { + if report.sent_invalid() { + pending.failed_peers.insert(request.peer); + } return if pending.columns.is_empty() { ColumnAttemptOutcome::Complete } else { @@ -2072,6 +2124,7 @@ fn drop_stored_columns(server: &mut P2PServer, block_root: H256) { .columns .retain(|&column| !store.has_data_column(slot, &block_root, column)); } + /// The beacon wire, or a refusal sent on `channel`. /// /// A beacon request reaching a lean node means a peer negotiated a protocol @@ -3744,7 +3797,16 @@ pub(crate) mod tests { "the new lookup's requests stay open" ); assert_eq!(pending.attempts, 1); - assert!(pending.failed_peers.is_empty()); + assert!( + pending.failed_peers.is_empty(), + "a failure may be what got the old lookup replaced" + ); + + // An invalid sidecar still writes its sender off for this root. + late.reply = ColumnReply::Answered(vec![(1, ColumnVerdict::Rejected)]); + settle_column_report(&mut pending, &late); + assert_eq!(pending.failed_peers, HashSet::from([old_peer])); + assert_eq!(pending.asked.len(), 2); // What such a request delivers is here all the same. late.reply = delivering(&[1, 2]); @@ -3955,6 +4017,66 @@ pub(crate) mod tests { assert!(metrics::data_column_fetch_failures_total("no_custodian") > before); } + #[tokio::test] + async fn past_the_tip_a_peer_is_still_asked_for_the_columns_it_has_not_missed() { + // It used to be passed over for every uncovered column once it had + // missed any one of them, so a column merged in after its short answer + // went to nobody and waited for the next slot's re-ask. + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(silent, None)]).await; + let root = H256::repeat_byte(0x59); + let mut pending = column_lookup(vec![1, 2], &[]); + pending.slot = Some(5); + pending.missed.insert((silent, 1)); + server.pending_column_requests.insert(root, pending); + + ask_unasked_columns(&mut server, root).await; + + assert_eq!( + server.pending_column_requests[&root].asked, + HashMap::from([(2, silent)]) + ); + } + + /// An empty fulu block at `slot`. + fn fulu_block_at(slot: u64) -> SignedBeaconBlock { + use ethlambda_types::beacon::containers::electra; + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + + #[tokio::test] + async fn a_lookup_reads_the_tip_from_its_held_block_in_the_store() { + // Nothing tells a lookup its block's slot: a held envelope's block is + // imported and a held fulu block is written as pending, so the store + // has it, and a peer that answered short at the tip is asked again. + let silent = PeerId::random(); + let mut server = beacon_server_with(&[(silent, None)]).await; + let now = server.wire.beacon().expect("a beacon wire").wall_slot(); + let root = H256::repeat_byte(0x5B); + server + .store + .insert_pending_block(root, fulu_block_at(now)) + .expect("stored"); + let mut pending = column_lookup(vec![1], &[]); + pending.missed.insert((silent, 1)); + server.pending_column_requests.insert(root, pending); + + ask_unasked_columns(&mut server, root).await; + + let pending = &server.pending_column_requests[&root]; + assert_eq!(pending.slot, Some(now)); + assert_eq!(pending.asked, HashMap::from([(1, silent)])); + } + #[tokio::test] async fn a_retry_asks_only_for_what_the_store_does_not_have() { // Gossip often delivers a column while a lookup started at the tip is From 4bae913a0f771a8b8219ecbdcbccce7643225677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:17:42 -0300 Subject: [PATCH 6/7] fix(p2p): count a column lookup whose every peer failed as peers_exhausted A lookup that had written off every connected peer (a failed request or an invalid sidecar) and had nobody left to ask counted as no_custodian, which says nobody holds the columns. Nobody was left to say. It now counts as peers_exhausted on lean_data_column_fetch_failures_total; the next slot's re-ask starts a fresh lookup that asks them again. --- crates/net/p2p/src/metrics.rs | 9 +++--- crates/net/p2p/src/req_resp/handlers.rs | 43 +++++++++++++++++++++++++ docs/metrics.md | 8 +++-- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index 0b7d2724e..1700f2ec6 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -504,10 +504,11 @@ static LEAN_DATA_COLUMN_FETCH_FAILURES_TOTAL: LazyLock = LazyLock /// Count one `DataColumnsByRoot` lookup this node gave up on. `reason` is /// `"no_peers"` (nothing connected to ask), `"max_retries"` (the retry ladder -/// ran out on columns some connected peer is known to custody) or -/// `"no_custodian"` (nobody connected was found holding the missing columns: -/// none is known to custody them, or every peer asked past the tip answered -/// without them). +/// ran out on columns some connected peer is known to custody), +/// `"peers_exhausted"` (every connected peer failed a request or sent an +/// invalid sidecar in this lookup) or `"no_custodian"` (nobody connected was +/// found holding the missing columns: none is known to custody them, or every +/// peer asked past the tip answered without them). pub fn inc_data_column_fetch_failure(reason: &str) { LEAN_DATA_COLUMN_FETCH_FAILURES_TOTAL .with_label_values(&[reason]) diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 9edcfae76..831bf4b30 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -2007,6 +2007,10 @@ enum Retirement { /// or every one asked past the tip answered without it), or the ladder /// ran out on columns no connected peer is known to custody. NobodyToAsk, + /// Nothing is on the wire, and every connected peer has been written off + /// in this lookup (a failed request or a refused sidecar), so whether one + /// of them holds what is missing was never learned. + PeersExhausted, /// Nothing is on the wire, and nothing is connected at all. NoPeers, } @@ -2043,6 +2047,17 @@ fn retire_column_lookup(server: &mut P2PServer, block_root: H256, why: Retiremen ); metrics::inc_data_column_fetch_failure("no_custodian"); } + // The old lookup cleared its written-off peers and asked them all + // again; the next re-ask's fresh lookup does that now, a slot later. + Retirement::PeersExhausted => { + let missing = pending.columns; + debug!( + %block_root, + ?missing, + "Every connected peer failed this lookup; leaving its columns to the next re-ask" + ); + metrics::inc_data_column_fetch_failure("peers_exhausted"); + } // Counted as `no_peers` where the send found nobody connected. Retirement::NoPeers => { debug!(%block_root, "No connected peers to ask for this lookup's columns"); @@ -2091,8 +2106,19 @@ pub(crate) async fn ask_unasked_columns(server: &mut P2PServer, block_root: H256 .get(&block_root) .is_some_and(|pending| !pending.asked.is_empty()); if !sent && !open { + let all_failed = server + .pending_column_requests + .get(&block_root) + .is_some_and(|pending| { + server + .connected_peers + .keys() + .all(|peer| pending.failed_peers.contains(peer)) + }); let why = if server.connected_peers.is_empty() { Retirement::NoPeers + } else if all_failed { + Retirement::PeersExhausted } else { Retirement::NobodyToAsk }; @@ -4038,6 +4064,23 @@ pub(crate) mod tests { ); } + #[tokio::test] + async fn a_lookup_every_peer_failed_ends_as_peers_exhausted() { + // Not "nobody holds the columns": nobody was left to say. + let custodian = PeerId::random(); + let mut server = beacon_server_with(&[(custodian, Some(vec![1]))]).await; + let root = H256::repeat_byte(0x5A); + let mut pending = column_lookup(vec![1], &[]); + pending.failed_peers.insert(custodian); + server.pending_column_requests.insert(root, pending); + let before = metrics::data_column_fetch_failures_total("peers_exhausted"); + + ask_unasked_columns(&mut server, root).await; + + assert!(server.pending_column_requests.is_empty()); + assert!(metrics::data_column_fetch_failures_total("peers_exhausted") > before); + } + /// An empty fulu block at `slot`. fn fulu_block_at(slot: u64) -> SignedBeaconBlock { use ethlambda_types::beacon::containers::electra; diff --git a/docs/metrics.md b/docs/metrics.md index b657b5f88..d5385e772 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -600,7 +600,7 @@ spec. | `lean_data_columns_stored_total` | Counter | Data column sidecars verified and written to `Table::DataColumns` | On the chain actor storing a sidecar the p2p layer verified | | | | `lean_data_columns_rejected_total` | Counter | Sidecars the p2p layer's chain checks dropped, by reason | On each drop in `beacon::column_checks`, except an already stored sidecar | reason=malformed,future_slot,finalized,not_after_parent,unknown_proposer,bad_signature,wrong_proposer,finalized_not_ancestor,parent_not_ready,inclusion_proof,kzg,internal | | | `lean_data_column_kzg_verify_seconds` | Histogram | Time spent batch-verifying one sidecar's cells against its own commitments | On each KZG batch verification, in gossip validation or the chain checks | | 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0 | -| `lean_data_column_fetch_failures_total` | Counter | `DataColumnsByRoot` lookups this node gave up on, by reason | On lookup abandonment | reason=no_peers,max_retries,no_custodian | | +| `lean_data_column_fetch_failures_total` | Counter | `DataColumnsByRoot` lookups this node gave up on, by reason | On lookup abandonment | reason=no_peers,max_retries,peers_exhausted,no_custodian | | | `lean_blocks_held_for_columns` | Gauge | Blocks held out of fork choice pending their custody columns | On every hold, release, and finality eviction of the held-block set | | | | `lean_sidecars_awaiting_parent` | Gauge | Sidecars parked until their block's parent has a post-state | On every park, replay, and finality eviction of the parked set | | | | `lean_envelopes_awaiting_block` | Gauge | Gloas execution payload envelopes held until their block is imported | On every hold, release, and finality eviction of the held envelopes | | | @@ -621,8 +621,10 @@ availability gate, and a healthy node returns it to zero within a slot or two of a hold: fetch or gossip should complete well inside the retry ladder `lean_data_column_fetch_failures_total` counts down. A gauge that sits above zero for several slots means either the fetch is failing — check whether -`lean_data_column_fetch_failures_total`'s `no_peers` or `max_retries` reason -is climbing — or no reachable peer actually serves the missing columns, which +`lean_data_column_fetch_failures_total`'s `no_peers`, `max_retries` or +`peers_exhausted` reason is climbing (the last: every connected peer failed a +request or sent an invalid sidecar in one lookup) — or no reachable peer +actually serves the missing columns, which the `no_custodian` reason counts: a lookup that ends without finding anyone connected holding the missing columns, because none is known to custody them or every peer asked after the block's slot answered without them. From f31e5211758b9647ea010656dd53a796cd3097d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:22:37 -0300 Subject: [PATCH 7/7] docs(beacon_wire): say the uncovered columns are assigned per column --- docs/beacon_wire.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index f8d8d2357..fd4e34afa 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -749,9 +749,10 @@ from `metadata/3`, requested once per connection right behind `Status`, and is seeded from the ENR `cgc` for a peer this node dialed; a peer that has supplied neither is not assumed to custody anything, and both paths send whatever no known custodian covers to such a peer rather than to one that has said what it -keeps. The by-root path asks one at a time; only when none is connected does -it ask a peer of known custody for those columns, once per lookup, since such -a peer may hold more than it custodies. At mainnet's +keeps. The by-root path gives as many of those columns as it can to one such +peer at a time; only a column none of them can be asked for goes to a peer of +known custody, once per lookup, since such a peer may hold more than it +custodies. At mainnet's `CUSTODY_REQUIREMENT` a peer holds 8 of 128 columns, so this is the difference between a request that can be answered and one that usually cannot.