From 06e3306088c31b6a060adba46ec6307280c5db73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:43:31 -0300 Subject: [PATCH 01/11] perf(blockchain): run the beacon state transition while the engine verdict is in flight With an execution client, an import waited for engine_newPayload and then ran the state transition, so the two summed. The request is now built on the actor and sent from a spawned task, and the actor computes the transition meanwhile, so the import costs the larger of the two. The speculative post-state is a local value (SpeculativeTransition) until on_block_speculative accepts it. on_block's checks run unchanged and in the same order; only the transition_block call is replaced, and only when the speculation was computed for this block root and parent root and the verdict is not Invalidated. on_block derives a valid() engine for every other verdict, so the speculation is the transition it would have run. Invalidated, held blocks (no verdict, non-candidate optimistic) and a failed or panicked request task behave as before. on_block keeps its signature as a wrapper, so spec-test runners and the replay harness (no engine) are untouched. Timings: the transition that overlapped the round trip is reported under stf, and engine reports only the wait left after it, so the rows still tile the end-to-end time. A discarded speculation leaves engine covering the overlap. lean_engine_speculation_total{outcome} counts used and discarded attempts. --- crates/blockchain/src/beacon_engine.rs | 48 +++ crates/blockchain/src/import_timing.rs | 55 ++- crates/blockchain/src/lib.rs | 133 +++++-- crates/blockchain/src/metrics.rs | 27 ++ .../src/beacon/fork_choice.rs | 364 ++++++++++++++++-- docs/beacon_engine.md | 34 ++ docs/benchmarking.md | 5 + docs/metrics.md | 5 +- 8 files changed, 604 insertions(+), 67 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 38f34c39e..b3733683b 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -178,6 +178,54 @@ pub fn verdict(status: &PayloadStatusV1) -> PayloadValidity { }) } +/// [`NewPayloadRequest`] with its own copy of the payload, so it can move into +/// a spawned task. +/// +/// The chain actor sends the question from a task of its own and runs the +/// block's state transition meanwhile (see `process_block`), and a task cannot +/// borrow the block the actor goes on using. Only the payload is copied: the +/// rest of the request is already owned, and the block itself, which also +/// carries the attestations and the other operations, is never cloned. +pub struct OwnedNewPayloadRequest { + execution_payload: containers::deneb::ExecutionPayload, + versioned_hashes: Vec, + parent_beacon_block_root: Root, + execution_requests: Vec>, +} + +impl NewPayloadRequest<'_> { + /// Copies the borrowed payload so the request outlives the block. + pub fn into_owned(self) -> OwnedNewPayloadRequest { + OwnedNewPayloadRequest { + execution_payload: self.execution_payload.clone(), + versioned_hashes: self.versioned_hashes, + parent_beacon_block_root: self.parent_beacon_block_root, + execution_requests: self.execution_requests, + } + } +} + +/// [`ask`] for a request already built with [`new_payload_request`], which is +/// how the actor sends it from a task while it computes the transition. +/// +/// The answer is read exactly as [`ask`] reads it: `Err` means no answer after +/// the client's whole retry ladder, and there is no `Ok(None)` because a +/// request existing means there was something to ask. +pub async fn ask_request( + client: &EngineClient, + request: OwnedNewPayloadRequest, +) -> Result { + let status = client + .new_payload( + &request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ) + .await?; + Ok(verdict(&status)) +} + /// Asks the execution client about `block`'s payload. /// /// `Ok(None)` means there was nothing to ask, which is not a failure: diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index 6625a472f..01ff46266 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -177,7 +177,9 @@ pub struct ImportTimings { pub columns_wait_end: Option, /// Beacon: the `engine_newPayload` round trip, including its retry ladder. - /// I/O wait, not work. + /// I/O wait, not work. When the state transition ran meanwhile and was + /// used, this pair starts where that transition ended and so covers only + /// the wait that remained; see [`Self::stf_spec_start`]. pub engine_start: Option, pub engine_end: Option, @@ -197,6 +199,21 @@ pub struct ImportTimings { pub stf_start: Option, pub stf_end: Option, + /// Beacon: a state transition computed while the execution client's + /// verdict was still in flight, and then used by the import. + /// + /// It runs before the `stf` pair above and overlaps the `engine_newPayload` + /// round trip, so it is neither inside `stf_start`..`stf_end` nor a + /// section of its own: the `stf` row adds it, since it is the same + /// state-transition work done earlier. Meanwhile `engine_start` is moved to + /// this pair's end, so the `engine` row reports only the wait that was left + /// once the transition finished, and the rows still sum to the end-to-end + /// time instead of counting the overlap twice. `None` whenever the import + /// ran its own transition, which is every import without an execution + /// client and every one whose speculation was discarded. + pub stf_spec_start: Option, + pub stf_spec_end: Option, + /// Beacon: the blocking hand-off of the post-state to the storage crate's /// background writer, which waits while the writer's queue is full. It /// lies inside the `stf` pair; only the report separates the two. @@ -376,6 +393,11 @@ impl ImportTimings { if let (Some(total), Some(wait)) = (stf.elapsed, writer_wait.elapsed) { stf.elapsed = Some(total.saturating_sub(wait)); } + if let (Some(total), Some(spec)) = + (stf.elapsed, span(self.stf_spec_start, self.stf_spec_end)) + { + stf.elapsed = Some(total + spec); + } vec![ Row::new("decode", self.decode_start, self.decode_end), Row::new("queue", self.queue_start, self.queue_end), @@ -897,6 +919,37 @@ mod tests { assert_eq!(row_elapsed(&timings, "writer_wait"), None); } + #[test] + fn an_overlapped_transition_is_charged_to_stf_and_not_to_engine() { + // A 100 ms round trip with a 70 ms transition running inside it, then + // 20 ms of import. Un-overlapped, that is 100 + 20 ms; overlapped, the + // engine row keeps the 30 ms that was left and stf gains the 70 ms. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.stf_spec_start, timings.stf_spec_end) = pair(base, 0, 70); + (timings.engine_start, timings.engine_end) = pair(base, 70, 30); + (timings.stf_start, timings.stf_end) = pair(base, 100, 20); + + assert_eq!( + row_elapsed(&timings, "engine"), + Some(Duration::from_millis(30)) + ); + assert_eq!( + row_elapsed(&timings, "stf"), + Some(Duration::from_millis(90)) + ); + let rows: Duration = timings + .rows() + .into_iter() + .filter_map(|row| row.elapsed) + .sum(); + assert_eq!( + rows, + Duration::from_millis(120), + "the rows still tile the span" + ); + } + #[test] fn a_writer_wait_longer_than_stf_saturates_stf_to_zero() { let base = Instant::now(); diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 251048b7a..53a6fef24 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1796,31 +1796,81 @@ impl BlockChainServer { // `is_data_available` runs before `state_transition`, so a // block about to be held for its custody columns is never one // this node asks an execution client about. + // + // With an execution client the question is sent from a task + // of its own and this actor computes the block's state + // transition while the answer is awaited, so the two cost + // `max(engine wait, transition)` instead of their sum. The + // result stays a local value: it reaches the store only + // through `fork_choice::on_block_speculative`, after every + // check `on_block` makes and only for a verdict that agrees + // with the transition it assumed (see `SpeculativeTransition`). + // Without an engine, or without a request to send (pre-merge + // forks, gloas), nothing overlaps and the path is the old one. timings.engine_start = Some(Instant::now()); - let validity = match &self.engine { - None => fork_choice::PayloadValidity::NotRequired, - Some(client) => match beacon_engine::ask(client, &beacon_block).await { - Ok(None) => fork_choice::PayloadValidity::NotRequired, - Ok(Some(validity)) => validity, - // No answer after the whole ladder. - // `optimistic-sync.md`: a consensus engine MUST NOT - // import the block and MUST NOT apply it to the fork - // choice store. Returning `Held` leaves the block out - // of the store without marking it imported, so the - // cascade does not treat it as having produced a - // post-state. - Err(err) => { - warn!( - %slot, - block_root = %ShortRoot(&block_root.0), - %err, - "No verdict from the execution client; not importing" - ); - metrics::inc_engine_no_verdict(); - timings.engine_end = Some(Instant::now()); - return (timings, Ok(ImportOutcome::Held)); + let parent_root = beacon_block.parent_root(); + let mut speculation: Option<( + fork_choice::SpeculativeTransition, + Instant, + Instant, + )> = None; + let verdict: Result, String> = + match &self.engine { + None => Ok(None), + Some(client) => { + match beacon_engine::new_payload_request(&beacon_block) + .map(|request| request.into_owned()) + { + None => Ok(None), + Some(request) => { + let client = client.clone(); + let task = tokio::spawn(async move { + beacon_engine::ask_request(&client, request).await + }); + let spec_start = Instant::now(); + let computed = fork_choice::speculate_transition( + &self.store, + &beacon_block, + &config, + &committees, + ); + let spec_end = Instant::now(); + speculation = computed.map(|spec| (spec, spec_start, spec_end)); + // A panicked or cancelled task is no + // answer, like an exhausted retry ladder. + match task.await { + Ok(Ok(validity)) => Ok(Some(validity)), + Ok(Err(err)) => Err(err.to_string()), + Err(err) => Err(err.to_string()), + } + } + } } - }, + }; + let validity = match verdict { + Ok(None) => fork_choice::PayloadValidity::NotRequired, + Ok(Some(validity)) => validity, + // No answer after the whole ladder. + // `optimistic-sync.md`: a consensus engine MUST NOT + // import the block and MUST NOT apply it to the fork + // choice store. Returning `Held` leaves the block out + // of the store without marking it imported, so the + // cascade does not treat it as having produced a + // post-state. + Err(err) => { + warn!( + %slot, + block_root = %ShortRoot(&block_root.0), + %err, + "No verdict from the execution client; not importing" + ); + metrics::inc_engine_no_verdict(); + if speculation.is_some() { + metrics::inc_engine_speculation("discarded_held"); + } + timings.engine_end = Some(Instant::now()); + return (timings, Ok(ImportOutcome::Held)); + } }; timings.engine_end = Some(Instant::now()); @@ -1829,7 +1879,6 @@ impl BlockChainServer { // answer, is not imported at all. if matches!(validity, fork_choice::PayloadValidity::Optimistic) { let current_slot = self.wall_clock_slot(); - let parent_root = beacon_block.parent_root(); if !fork_choice::is_optimistic_candidate_block( &self.store, current_slot, @@ -1844,21 +1893,55 @@ impl BlockChainServer { block and it is not an optimistic candidate" ); metrics::inc_engine_not_optimistic_candidate(); + if speculation.is_some() { + metrics::inc_engine_speculation("discarded_held"); + } return (timings, Ok(ImportOutcome::Held)); } } + // The speculation is only worth handing on for a verdict its + // `valid()` engine agrees with. `INVALIDATED` makes `on_block` + // run the transition itself with `invalid()`, exactly as + // before. A discarded speculation leaves `engine_start` where + // it was, so the `engine` row covers the whole overlap and + // the wasted transition is not hidden from the tree. + let speculation = match speculation { + Some((spec, spec_start, spec_end)) => { + if !spec.is_for(block_root, parent_root) { + metrics::inc_engine_speculation("discarded_mismatch"); + None + } else if matches!( + validity, + fork_choice::PayloadValidity::Invalidated { .. } + ) { + metrics::inc_engine_speculation("discarded_invalidated"); + None + } else { + metrics::inc_engine_speculation("used"); + // The wait that remained once the transition was + // done; see `ImportTimings::stf_spec_start`. + timings.engine_start = Some(spec_end); + timings.stf_spec_start = Some(spec_start); + timings.stf_spec_end = Some(spec_end); + Some(spec) + } + } + None => None, + }; + // Drop any hand-off span left by an insert outside this call, // so the one taken below is this block's alone. let _ = self.store.take_state_handoff(); timings.stf_start = Some(Instant::now()); - let imported = fork_choice::on_block( + let imported = fork_choice::on_block_speculative( &mut self.store, beacon_block, &config, &evidence, &validity, &committees, + speculation, ); timings.stf_end = Some(Instant::now()); (timings.writer_wait_start, timings.writer_wait_end) = diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 34c67398f..e0cd83a67 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1507,6 +1507,33 @@ pub fn inc_engine_no_verdict() { LEAN_ENGINE_NO_VERDICT_TOTAL.inc(); } +/// Speculative state transitions, by what became of them. +/// +/// With an execution client, the chain actor runs a block's state transition +/// while the `engine_newPayload` round trip is in flight. `outcome` is `used` +/// (handed to `fork_choice::on_block` with a verdict that lets it stand in for +/// the transition; a block that then fails one of `on_block`'s earlier checks +/// still counts, since the work was spent either way), `discarded_invalidated` +/// (the verdict was `INVALIDATED`, so `on_block` runs its own), `discarded_held` +/// (no verdict, or a non-candidate `SYNCING`/`ACCEPTED`, so the block is not +/// imported) or `discarded_mismatch` (computed for another root or parent; +/// not expected from the actor). A high `discarded_*` share means the overlap +/// is buying nothing. +pub fn inc_engine_speculation(outcome: &str) { + static LEAN_ENGINE_SPECULATION_TOTAL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_counter_vec!( + "lean_engine_speculation_total", + "Speculative state transitions run during the engine round trip, by outcome", + &["outcome"] + ) + .unwrap() + }); + LEAN_ENGINE_SPECULATION_TOTAL + .with_label_values(&[outcome]) + .inc(); +} + /// Blocks not imported because the execution client has not validated them and /// they do not qualify for an optimistic import. /// diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 02c959376..cd970a3d6 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4777,6 +4777,119 @@ pub fn on_block( blob_evidence: &DataAvailability, payload_validity: &PayloadValidity, committees: &CommitteeCache, +) -> Result<()> { + on_block_speculative( + store, + signed_block, + config, + blob_evidence, + payload_validity, + committees, + None, + ) +} + +/// The state transition of one block, computed before the execution client's +/// verdict on its payload is known. +/// +/// Opaque on purpose: the only thing a holder can do with one is hand it to +/// [`on_block_speculative`], which decides whether it may stand in for the +/// transition `on_block` would otherwise run. Until then it is a local value; +/// nothing here has reached the store, and dropping it costs only the work. +/// +/// # Why it is the transition `on_block` would run +/// +/// `on_block` reads the verdict in one place, to pick the +/// [`stf::ExecutionEngine`] the transition consults: `invalid()` for +/// [`PayloadValidity::Invalidated`], `valid()` for every other verdict +/// (`NotRequired`, `Validated` and `Optimistic` alike). So a transition run +/// with `valid()` is byte for byte the one `on_block` runs for every verdict +/// except `Invalidated`, and for that one the speculation is simply not used. +/// +/// The inputs are the same too. [`transition_block`] takes the store by shared +/// reference and reads two things off it: the epoch-boundary state the +/// precompute worker cached, and, through [`cache_boundary_balances`], the +/// boundary balance cache it fills. Both are keyed by content (a checkpoint, +/// that is an epoch and a block root), so what they hold does not depend on +/// when they are consulted. Everything else is the parent's post-state, the +/// block, the config and the committee and active-balance caches, which are +/// content-keyed as well. The chain actor handles one message at a time, so +/// nothing mutates the store between this computation and `on_block`'s own. +pub struct SpeculativeTransition { + block_root: Root, + parent_root: Root, + state: BeaconState, + outcome: Result<()>, +} + +impl SpeculativeTransition { + /// Whether this was computed for the block with this root and parent root. + /// + /// [`on_block_speculative`] makes the same comparison before using one; a + /// caller asks it first only to tell a mismatch from a use when counting. + pub fn is_for(&self, block_root: Root, parent_root: Root) -> bool { + self.block_root == block_root && self.parent_root == parent_root + } +} + +/// Runs `signed_block`'s state transition against its parent's post-state with +/// the execution engine assumed valid, without touching the store. +/// +/// `None` when the parent has no post-state, the same condition under which +/// `on_block` fails with `block.parent_root in store.block_states`; the caller +/// then gets that failure from `on_block` itself. The parent state is looked up +/// exactly as `on_block` looks it up. +/// +/// None of `on_block`'s own checks run here. They run, unchanged and in the +/// same order, when the speculation is handed to [`on_block_speculative`], so a +/// block that fails one of them is rejected exactly as it is today and the +/// speculation is dropped. +pub fn speculate_transition( + store: &Store, + signed_block: &SignedBeaconBlock, + config: &Config, + committees: &CommitteeCache, +) -> Option { + let block_root = signed_block.message_hash_tree_root(); + let parent_root = signed_block.parent_root(); + let parent_state = store.get_state(&parent_root).expect("get")?; + let active_balances = store.active_balance_cache(); + let (state, outcome) = transition_block( + store, + &parent_state, + parent_root, + signed_block, + true, + config, + &stf::ExecutionEngine::valid(), + committees, + &active_balances, + ); + Some(SpeculativeTransition { + block_root, + parent_root, + state, + outcome, + }) +} + +/// [`on_block`] with an optional transition computed ahead of time. +/// +/// Every check `on_block` makes runs unchanged and in the same order. The only +/// difference is the one `transition_block` call: when `speculation` was +/// computed for this block's own root and parent root, and the verdict is not +/// `Invalidated` (the one verdict the speculation's `valid()` engine would +/// disagree with), its result stands in for that call. In every other case it +/// is dropped and the transition runs here as it always has, so the +/// invalidation path in particular is the one `on_block` has today. +pub fn on_block_speculative( + store: &mut Store, + signed_block: SignedBeaconBlock, + config: &Config, + blob_evidence: &DataAvailability, + payload_validity: &PayloadValidity, + committees: &CommitteeCache, + speculation: Option, ) -> Result<()> { let block_root = signed_block.message_hash_tree_root(); let parent_root = signed_block.parent_root(); @@ -4949,17 +5062,24 @@ pub fn on_block( // The store's own cache, so every block of this chain shares one entry per // epoch; see `ActiveBalanceCacheExt`. let active_balances = store.active_balance_cache(); - let (mut state, transition) = transition_block( - store, - &parent_state, - parent_root, - &signed_block, - true, - config, - &engine, - committees, - &active_balances, - ); + let usable = speculation.filter(|speculation| { + speculation.is_for(block_root, parent_root) + && !matches!(payload_validity, PayloadValidity::Invalidated { .. }) + }); + let (mut state, transition) = match usable { + Some(speculation) => (speculation.state, speculation.outcome), + None => transition_block( + store, + &parent_state, + parent_root, + &signed_block, + true, + config, + &engine, + committees, + &active_balances, + ), + }; // `optimistic-sync.md`: a block deemed `INVALIDATED` MUST NOT be included // in the canonical chain. That is stated here, on the verdict, rather than @@ -7111,34 +7231,13 @@ mod tests { assert!(store.is_err()); } - /// [`on_block`]'s early-return guard must read `Store::has_state`, not - /// `Store::has_block`: a block the actor is holding for missing data - /// columns is written with - /// [`Store::insert_pending_block`](ethlambda_storage::Store::insert_pending_block) - /// before it ever imports, which stores the whole signed block in one - /// `BlockHeaders` row (so `has_block` answers true) but writes no - /// `LiveChain` entry (so [`Store::block_index`](ethlambda_storage::Store::block_index), - /// a `LiveChain` scan, does not name it). Guarding on `has_block` instead - /// of `has_state` made `on_block` answer `Ok(())` for a held block - /// without importing it, so every later delivery naming it as an - /// ancestor found it "already known" and gave up without importing - /// anything either -- forever, since nothing ever gave the root a - /// post-state. + /// A store anchored one slot before a genuinely valid child block, with the + /// clock at that block's slot, and the block itself. /// - /// Delivered timely (the first block for its slot, before the - /// attestation deadline), so this also proves `on_block`'s pre-import - /// `compute_head` call (walking `Store::block_index`, for the new - /// proposer-boost dependent-root gate) tolerates a held block sitting - /// unindexed in the store: that walk never sees it, being a `LiveChain` - /// scan, so it is untouched by the held block's absent post-state. - /// - /// Builds a genuinely valid child block (correct proposer, RANDAO reveal, - /// and state root, all real BLS signatures over - /// [`test_state::secret_key_for`]'s key) rather than the zero-signature - /// [`block`] helper, since this exercises the real, signature-checking - /// [`on_block`], not a fixture harness that can skip verification. - #[test] - fn on_block_imports_a_block_already_persisted_with_no_post_state() { + /// Valid means correct proposer, RANDAO reveal and state root, all real BLS + /// signatures over [`test_state::secret_key_for`]'s key, so the real, + /// signature-checking [`on_block`] accepts it. + fn store_with_valid_child() -> (Config, Store, SignedBeaconBlock) { let config = Config::active(); let (anchor_state, anchor_block) = anchor_pair_with(1); let anchor_root = anchor_block.message_hash_tree_root(); @@ -7204,7 +7303,6 @@ mod tests { .to_bytes(), ); let signed_block = draft; - let block_root = signed_block.message_hash_tree_root(); // The store holds the child block already, exactly the shape // `hold_block_for_columns` leaves it in: one `BlockHeaders` row (so @@ -7229,6 +7327,39 @@ mod tests { config.seconds_per_slot * target_slot, )) .expect("set time"); + (config, store, signed_block) + } + + /// [`on_block`]'s early-return guard must read `Store::has_state`, not + /// `Store::has_block`: a block the actor is holding for missing data + /// columns is written with + /// [`Store::insert_pending_block`](ethlambda_storage::Store::insert_pending_block) + /// before it ever imports, which stores the whole signed block in one + /// `BlockHeaders` row (so `has_block` answers true) but writes no + /// `LiveChain` entry (so [`Store::block_index`](ethlambda_storage::Store::block_index), + /// a `LiveChain` scan, does not name it). Guarding on `has_block` instead + /// of `has_state` made `on_block` answer `Ok(())` for a held block + /// without importing it, so every later delivery naming it as an + /// ancestor found it "already known" and gave up without importing + /// anything either -- forever, since nothing ever gave the root a + /// post-state. + /// + /// Delivered timely (the first block for its slot, before the + /// attestation deadline), so this also proves `on_block`'s pre-import + /// `compute_head` call (walking `Store::block_index`, for the new + /// proposer-boost dependent-root gate) tolerates a held block sitting + /// unindexed in the store: that walk never sees it, being a `LiveChain` + /// scan, so it is untouched by the held block's absent post-state. + /// + /// Builds a genuinely valid child block (correct proposer, RANDAO reveal, + /// and state root, all real BLS signatures over + /// [`test_state::secret_key_for`]'s key) rather than the zero-signature + /// [`block`] helper, since this exercises the real, signature-checking + /// [`on_block`], not a fixture harness that can skip verification. + #[test] + fn on_block_imports_a_block_already_persisted_with_no_post_state() { + let (config, mut store, signed_block) = store_with_valid_child(); + let block_root = signed_block.message_hash_tree_root(); store .insert_pending_block(block_root, signed_block.clone()) .expect("insert"); @@ -7260,6 +7391,161 @@ mod tests { ); } + /// Imports the valid child with `validity` and `speculation` and reports + /// everything an import writes that a later reader could tell apart. + fn import_valid_child( + validity: &PayloadValidity, + speculate: impl FnOnce(&Store, &SignedBeaconBlock, &Config) -> Option, + ) -> (Result<()>, Store, Root) { + let (config, mut store, signed_block) = store_with_valid_child(); + let block_root = signed_block.message_hash_tree_root(); + let committees = CommitteeCache::default(); + let speculation = speculate(&store, &signed_block, &config); + let result = on_block_speculative( + &mut store, + signed_block, + &config, + &DataAvailability::NotRequired, + validity, + &committees, + speculation, + ); + (result, store, block_root) + } + + fn honest_speculation( + store: &Store, + block: &SignedBeaconBlock, + config: &Config, + ) -> Option { + speculate_transition(store, block, config, &CommitteeCache::default()) + } + + /// What the store holds after an import, as comparable values. + fn observed( + store: &mut Store, + block_root: Root, + config: &Config, + ) -> ( + Option, + Option<[bool; constants::NUM_BLOCK_TIMELINESS_DEADLINES]>, + Root, + Root, + Vec, + ) { + let post_state_root = store + .get_state(&block_root) + .expect("get") + .map(|state| state.hash_tree_root()); + let mut roots: Vec = store.block_index().keys().copied().collect(); + roots.sort(); + ( + post_state_root, + store.block_timeliness(&block_root), + store.proposer_boost_root(), + get_head(store, config).expect("head"), + roots, + ) + } + + #[test] + fn a_matching_speculation_leaves_the_store_as_importing_without_one() { + let config = Config::active(); + let (plain_result, mut plain, root) = + import_valid_child(&PayloadValidity::NotRequired, |_, _, _| None); + plain_result.expect("the plain import succeeds"); + for validity in [ + PayloadValidity::NotRequired, + PayloadValidity::Validated, + PayloadValidity::Optimistic, + ] { + let (result, mut speculated, speculated_root) = + import_valid_child(&validity, honest_speculation); + result.expect("the speculated import succeeds"); + assert_eq!(speculated_root, root); + assert_eq!( + observed(&mut speculated, root, &config), + observed(&mut plain, root, &config), + "{validity:?}: head, post-state root, timeliness, proposer boost and the \ + block index must be identical" + ); + } + } + + /// A speculation whose outcome is a marker nothing else produces, so the + /// result of the import says whether the speculation or a fresh transition + /// supplied it. + const POISON: &str = "poisoned speculation"; + + fn poisoned( + store: &Store, + block: &SignedBeaconBlock, + config: &Config, + ) -> Option { + let mut speculation = honest_speculation(store, block, config)?; + speculation.outcome = Err(Error::SpecAssert(POISON)); + Some(speculation) + } + + #[test] + fn a_matching_speculation_is_the_transition_on_block_uses() { + let (result, store, root) = import_valid_child(&PayloadValidity::Validated, poisoned); + assert_eq!(failed_assertion(result), Some(POISON)); + assert!( + !store.has_state(&root).unwrap(), + "a failed import writes nothing" + ); + } + + #[test] + fn a_speculation_for_another_root_or_parent_is_ignored() { + for tamper in [0u8, 1] { + let (result, store, root) = + import_valid_child(&PayloadValidity::NotRequired, |store, block, config| { + let mut speculation = poisoned(store, block, config)?; + match tamper { + 0 => speculation.block_root = Root::repeat_byte(0xee), + _ => speculation.parent_root = Root::repeat_byte(0xee), + } + Some(speculation) + }); + result.expect("the stale speculation is dropped and the real transition runs"); + assert!(store.has_state(&root).unwrap()); + } + } + + #[test] + fn an_invalidated_verdict_ignores_the_speculation_and_matches_today() { + let invalidated = PayloadValidity::Invalidated { + latest_valid_hash: None, + }; + let (without, plain_store, root) = import_valid_child(&invalidated, |_, _, _| None); + let (with_honest, honest_store, _) = import_valid_child(&invalidated, honest_speculation); + let (with_poison, poison_store, _) = import_valid_child(&invalidated, poisoned); + let without = format!("{:?}", without.expect_err("an invalidated block fails")); + assert_eq!(format!("{:?}", with_honest.expect_err("fails")), without); + assert_eq!( + format!("{:?}", with_poison.expect_err("fails")), + without, + "the speculation must not supply the transition under Invalidated" + ); + for store in [&plain_store, &honest_store, &poison_store] { + assert!(!store.has_state(&root).unwrap()); + } + } + + #[test] + fn no_speculation_without_a_parent_state() { + let (config, store, mut block) = store_with_valid_child(); + let SignedBeaconBlock::Phase0(inner) = &mut block else { + unreachable!("phase0"); + }; + inner.message.parent_root = Root::repeat_byte(0xcd); + assert!( + speculate_transition(&store, &block, &config, &CommitteeCache::default()).is_none() + ); + } + #[test] fn the_commitments_subtree_index_is_the_bodys_own_position() { // The generalized index the fixture states, minus the offset of a tree diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 1aa13b184..d5d1502df 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -91,6 +91,40 @@ already `INVALIDATED`, which costs one merkleization on a path that should never run and buys the failure arriving where a reviewer checks this code against the specification. +### The transition runs while the answer is in flight + +With an execution client, `process_block` does not wait idle for the answer. It +builds the request on the actor (copying only the payload, never the block), +sends it from a spawned task so the HTTP exchange keeps progressing, and runs +`fork_choice::speculate_transition` on the actor meanwhile: the block's state +transition against its parent's post-state, with the engine assumed valid. The +import then costs `max(engine wait, transition)` instead of their sum. + +Why that is the transition `on_block` would run: `on_block` derives the +`ExecutionEngine` from the verdict in one place, `invalid()` for `INVALIDATED` +and `valid()` for `NOT_REQUIRED`, `VALID` and optimistic alike, and the +transition reads nothing else off the store except content-keyed caches. What +keeps it harmless: + +- The result is a local value (`SpeculativeTransition`). Nothing reaches the + store, a cache that is not content-addressed, or the network until + `on_block_speculative` accepts it, and that runs every check `on_block` makes, + unchanged and in the same order. +- It stands in for the transition only when it was computed for this block's + root and parent root and the verdict is not `INVALIDATED`. For `INVALIDATED` + it is dropped and `on_block` recomputes with `invalid()`, so the invalidation + path is the one described above, unchanged. +- A block that ends up held (no verdict, or `SYNCING`/`ACCEPTED` for a block + that is not an optimistic candidate) drops it. The attempt is counted in + `lean_engine_speculation_total{outcome}`. +- A panicked or cancelled request task is treated as no verdict, like an + exhausted retry ladder. + +The overlap shows in the import timing tree: the transition that ran during the +round trip is reported under `stf`, and `engine` reports only the wait that was +left once it finished. Without an execution client, and for forks that have no +request to send (pre-merge, gloas), nothing overlaps and the path is the old one. + ## The verdicts `optimistic-sync.md` groups the five wire statuses into three outcomes: diff --git a/docs/benchmarking.md b/docs/benchmarking.md index ae50db389..088faa3ad 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -277,6 +277,11 @@ is state-transition work and the two phases do not overlap. When blocks import faster than the writer persists them, the wait shows up here rather than in `stf`. +With an execution client the beacon `stf` row also includes a state +transition that ran while `engine_newPayload` was in flight, and `engine` then +reports only the wait left after it. A replay has no client, so neither applies +to its numbers. + ### What is excluded, and why Replay runs nothing that a corpus already answers for: diff --git a/docs/metrics.md b/docs/metrics.md index 1ea07953f..441abd8c5 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -262,6 +262,7 @@ Series appear as blocks arrive rather than being seeded, since a third of the ph | Name | Type | Usage | Sample collection event | Labels | Buckets | |------|------|-------|-------------------------|--------|---------| | `lean_block_import_phase_seconds` | Histogram | Time one section of a block's import, or of the arrival carrying it, took | Per section that ran, on each import, hold or failure | phase (see below); source=gossip,sync | 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | +| `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | | `lean_block_import_cascade_blocks` | Histogram | Blocks one arrival put through the import path (attempts, so a block that ends held or pended counts) | Once per arriving block message | | 1, 2, 3, 5, 8, 16, 32, 64, 128 | Per-block phases, in the order a block crosses them, plus `total` for a completed import: @@ -277,10 +278,10 @@ Per-block phases, in the order a block crosses them, plus `total` for a complete | `cascade_wait` | Between the parent's import finishing and this block being popped off the cascade queue | both | | `da_check` | The custody-column availability check itself, not the wait | beacon | | `columns_wait` | Held because custody columns had not all arrived | beacon | -| `engine` | The `engine_newPayload` round trip, including its retry ladder | beacon | +| `engine` | The `engine_newPayload` round trip, including its retry ladder. When the state transition ran meanwhile and was used, only the wait left after it finished | beacon | | `verify_struct` | Participant bounds checks and pubkey resolution | lean | | `verify_crypto` | The leanVM multi-message aggregate verification | lean | -| `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it | both | +| `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it. It includes a transition that ran during the `engine` round trip | both | | `writer_wait` | Beacon only: the importer's blocking hand-off of the post-state to the storage crate's background writer, which waits while the writer's queue is full. Subtracted from `stf`, so `stf` and `writer_wait` do not overlap | beacon | | `db_write` | The block write and handing the post-state off to the storage crate's background writer; the state's own encode/diff/commit cost is `lean_state_write_seconds` instead, not this row | lean | | `fc_head` | `update_head` | lean | From b95be16bc4880e5318f871099f4ec13921e05dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:49:10 -0300 Subject: [PATCH 02/11] perf(blockchain): time the payload's execution apart from the import's engine wait The import's engine section is meant to say how long the consensus block's import waited on the execution client, but it also carried the execution client's own time whenever nothing hid it: the whole round trip for a discarded speculation, and a few microseconds of scheduling noise for every block that had nothing to ask. Its meaning is now only the wait left once the import's own state transition was done. It is a recorded zero when there was no request to send (no execution client, or a block with no payload: pre-merge or gloas) and when the answer was already in once the transition finished, so it is non-zero only when the execution client was slower than the transition, or never answered. The speculative transition is charged to stf whether the import used it or not, so the rows still sum to the end-to-end time: a held block keeps the transition it paid for, and an INVALIDATED verdict shows its transition twice. How long the execution client took is now its own histogram, lean_engine_new_payload_seconds{outcome}, observed in ask_request from the request to the answer, retry ladder included, with outcome valid, optimistic, invalidated or no_verdict. --- crates/blockchain/src/beacon_engine.rs | 28 ++++++- crates/blockchain/src/import_timing.rs | 72 +++++++++++++---- crates/blockchain/src/lib.rs | 107 ++++++++++++++----------- crates/blockchain/src/metrics.rs | 33 ++++++++ docs/beacon_engine.md | 10 ++- docs/benchmarking.md | 13 +-- docs/metrics.md | 7 +- 7 files changed, 193 insertions(+), 77 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index b3733683b..14884b293 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -19,6 +19,8 @@ //! Gloas is the exception: its payload is revealed in an envelope, so its //! request ([`gloas_new_payload_request`]) needs the block and the envelope. +use std::time::Instant; + use ethlambda_engine::types::PayloadStatusValue; use ethlambda_engine::{EngineClient, EngineError, PayloadStatusV1}; use ethlambda_state_transition::beacon::containers::{self, SignedBeaconBlock}; @@ -211,19 +213,39 @@ impl NewPayloadRequest<'_> { /// The answer is read exactly as [`ask`] reads it: `Err` means no answer after /// the client's whole retry ladder, and there is no `Ok(None)` because a /// request existing means there was something to ask. +/// +/// The time from sending the request to the answer is observed as +/// `lean_engine_new_payload_seconds`, under the answer's [`outcome_label`]. pub async fn ask_request( client: &EngineClient, request: OwnedNewPayloadRequest, ) -> Result { - let status = client + let sent = Instant::now(); + let answer = client .new_payload( &request.execution_payload, &request.versioned_hashes, request.parent_beacon_block_root, &request.execution_requests, ) - .await?; - Ok(verdict(&status)) + .await + .map(|status| verdict(&status)); + crate::metrics::observe_engine_new_payload(outcome_label(&answer), sent.elapsed()); + answer +} + +/// The `outcome` an answer from [`ask_request`] is counted under. +/// +/// `not_required` is listed for completeness: a status from the execution +/// client always reads as one of the other three verdicts. +pub fn outcome_label(answer: &Result) -> &'static str { + match answer { + Ok(PayloadValidity::Validated) => "valid", + Ok(PayloadValidity::Optimistic) => "optimistic", + Ok(PayloadValidity::Invalidated { .. }) => "invalidated", + Ok(PayloadValidity::NotRequired) => "not_required", + Err(_) => "no_verdict", + } } /// Asks the execution client about `block`'s payload. diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index 01ff46266..e3b0daba8 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -176,10 +176,18 @@ pub struct ImportTimings { pub columns_wait_start: Option, pub columns_wait_end: Option, - /// Beacon: the `engine_newPayload` round trip, including its retry ladder. - /// I/O wait, not work. When the state transition ran meanwhile and was - /// used, this pair starts where that transition ended and so covers only - /// the wait that remained; see [`Self::stf_spec_start`]. + /// Beacon: how long the import sat waiting for the execution client's + /// verdict with no consensus work of its own left to do. I/O wait, not + /// work. + /// + /// The request is sent from a task of its own while the state transition + /// runs (see [`Self::stf_spec_start`]), so this pair starts where that + /// transition ended and ends when the answer arrived. Both are the same + /// instant, a recorded zero, when there was no request to send (no + /// execution client, or a block with no payload in it: pre-merge or + /// gloas) or when the answer was already in once the transition finished. + /// How long the execution client itself took is + /// `lean_engine_new_payload_seconds`, not this. pub engine_start: Option, pub engine_end: Option, @@ -200,17 +208,18 @@ pub struct ImportTimings { pub stf_end: Option, /// Beacon: a state transition computed while the execution client's - /// verdict was still in flight, and then used by the import. + /// verdict was still in flight, whether the import then used it or not. /// /// It runs before the `stf` pair above and overlaps the `engine_newPayload` /// round trip, so it is neither inside `stf_start`..`stf_end` nor a - /// section of its own: the `stf` row adds it, since it is the same - /// state-transition work done earlier. Meanwhile `engine_start` is moved to - /// this pair's end, so the `engine` row reports only the wait that was left - /// once the transition finished, and the rows still sum to the end-to-end - /// time instead of counting the overlap twice. `None` whenever the import - /// ran its own transition, which is every import without an execution - /// client and every one whose speculation was discarded. + /// section of its own: the `stf` row adds it, since it is state-transition + /// work. A discarded one is added too, so the transition an `INVALIDATED` + /// verdict makes `on_block` run again shows as the second transition it + /// is, and a held block still shows the one it paid for. The `engine` pair + /// starts at this pair's end, so the rows still sum to the end-to-end time + /// instead of counting the overlap twice. `None` whenever no request was + /// sent, which is every import without an execution client and every + /// block with no payload to ask about. pub stf_spec_start: Option, pub stf_spec_end: Option, @@ -393,10 +402,11 @@ impl ImportTimings { if let (Some(total), Some(wait)) = (stf.elapsed, writer_wait.elapsed) { stf.elapsed = Some(total.saturating_sub(wait)); } - if let (Some(total), Some(spec)) = - (stf.elapsed, span(self.stf_spec_start, self.stf_spec_end)) - { - stf.elapsed = Some(total + spec); + // Also when the import stopped before running its own transition (a + // held block), so the speculation it paid for is not lost. + if let Some(spec) = span(self.stf_spec_start, self.stf_spec_end) { + stf.elapsed = Some(stf.elapsed.unwrap_or_default() + spec); + stf.end = stf.end.or(self.stf_spec_end); } vec![ Row::new("decode", self.decode_start, self.decode_end), @@ -950,6 +960,36 @@ mod tests { ); } + #[test] + fn a_held_block_still_charges_its_speculation_to_stf() { + // No verdict: the 70 ms transition ran, then 300 ms of waiting, then + // the block was held without a transition of its own. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.stf_spec_start, timings.stf_spec_end) = pair(base, 0, 70); + (timings.engine_start, timings.engine_end) = pair(base, 70, 300); + + assert_eq!( + row_elapsed(&timings, "stf"), + Some(Duration::from_millis(70)) + ); + assert_eq!( + row_elapsed(&timings, "engine"), + Some(Duration::from_millis(300)) + ); + } + + #[test] + fn an_engine_with_nothing_to_wait_for_is_a_recorded_zero() { + // No request sent, or the answer was in before the transition ended: + // the row is present and zero, so every beacon block observes it. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.engine_start, timings.engine_end) = (Some(base), Some(base)); + + assert_eq!(row_elapsed(&timings, "engine"), Some(Duration::ZERO)); + } + #[test] fn a_writer_wait_longer_than_stf_saturates_stf_to_zero() { let base = Instant::now(); diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 53a6fef24..1bfd4a064 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1807,46 +1807,67 @@ impl BlockChainServer { // with the transition it assumed (see `SpeculativeTransition`). // Without an engine, or without a request to send (pre-merge // forks, gloas), nothing overlaps and the path is the old one. - timings.engine_start = Some(Instant::now()); + // + // The `engine` section is what the import waited once it had + // nothing left to do, so it starts as a recorded zero and only + // a request still unanswered when the transition is done + // widens it. The execution client's own time is + // `lean_engine_new_payload_seconds`, observed in + // `ask_request`. + let no_wait = Instant::now(); + (timings.engine_start, timings.engine_end) = (Some(no_wait), Some(no_wait)); let parent_root = beacon_block.parent_root(); - let mut speculation: Option<( - fork_choice::SpeculativeTransition, - Instant, - Instant, - )> = None; - let verdict: Result, String> = - match &self.engine { - None => Ok(None), - Some(client) => { - match beacon_engine::new_payload_request(&beacon_block) - .map(|request| request.into_owned()) - { - None => Ok(None), - Some(request) => { - let client = client.clone(); - let task = tokio::spawn(async move { - beacon_engine::ask_request(&client, request).await - }); - let spec_start = Instant::now(); - let computed = fork_choice::speculate_transition( - &self.store, - &beacon_block, - &config, - &committees, - ); - let spec_end = Instant::now(); - speculation = computed.map(|spec| (spec, spec_start, spec_end)); - // A panicked or cancelled task is no - // answer, like an exhausted retry ladder. - match task.await { - Ok(Ok(validity)) => Ok(Some(validity)), - Ok(Err(err)) => Err(err.to_string()), - Err(err) => Err(err.to_string()), - } + let mut speculation: Option = None; + let verdict: Result, String> = match &self + .engine + { + None => Ok(None), + Some(client) => { + match beacon_engine::new_payload_request(&beacon_block) + .map(|request| request.into_owned()) + { + None => Ok(None), + Some(request) => { + let client = client.clone(); + let task = tokio::spawn(async move { + let answer = beacon_engine::ask_request(&client, request).await; + (answer, Instant::now()) + }); + let spec_start = Instant::now(); + let computed = fork_choice::speculate_transition( + &self.store, + &beacon_block, + &config, + &committees, + ); + let spec_end = Instant::now(); + // Charged to `stf` whether or not the + // import ends up using it: it is + // state-transition work either way. + if computed.is_some() { + timings.stf_spec_start = Some(spec_start); + timings.stf_spec_end = Some(spec_end); } + speculation = computed; + // A panicked or cancelled task is no + // answer, like an exhausted retry ladder. + let (answer, answered_at) = match task.await { + Ok((answer, answered_at)) => ( + answer.map(Some).map_err(|err| err.to_string()), + answered_at, + ), + Err(err) => (Err(err.to_string()), Instant::now()), + }; + // The wait begins once the transition is + // done; an answer already in by then + // leaves it at zero. + timings.engine_start = Some(spec_end); + timings.engine_end = Some(answered_at.max(spec_end)); + answer } } - }; + } + }; let validity = match verdict { Ok(None) => fork_choice::PayloadValidity::NotRequired, Ok(Some(validity)) => validity, @@ -1868,11 +1889,9 @@ impl BlockChainServer { if speculation.is_some() { metrics::inc_engine_speculation("discarded_held"); } - timings.engine_end = Some(Instant::now()); return (timings, Ok(ImportOutcome::Held)); } }; - timings.engine_end = Some(Instant::now()); // An optimistic import is only permitted for a block that // qualifies. A block that does not, and got a NOT_VALIDATED @@ -1903,11 +1922,10 @@ impl BlockChainServer { // The speculation is only worth handing on for a verdict its // `valid()` engine agrees with. `INVALIDATED` makes `on_block` // run the transition itself with `invalid()`, exactly as - // before. A discarded speculation leaves `engine_start` where - // it was, so the `engine` row covers the whole overlap and - // the wasted transition is not hidden from the tree. + // before. A discarded speculation stays in the `stf` row, so + // the transition run twice shows as twice. let speculation = match speculation { - Some((spec, spec_start, spec_end)) => { + Some(spec) => { if !spec.is_for(block_root, parent_root) { metrics::inc_engine_speculation("discarded_mismatch"); None @@ -1919,11 +1937,6 @@ impl BlockChainServer { None } else { metrics::inc_engine_speculation("used"); - // The wait that remained once the transition was - // done; see `ImportTimings::stf_spec_start`. - timings.engine_start = Some(spec_end); - timings.stf_spec_start = Some(spec_start); - timings.stf_spec_end = Some(spec_end); Some(spec) } } diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index e0cd83a67..e801ee6b2 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1534,6 +1534,39 @@ pub fn inc_engine_speculation(outcome: &str) { .inc(); } +/// How long the execution client took to answer `engine_newPayload` for a +/// block's payload, from sending the request to its answer, retry ladder +/// included. +/// +/// The import computes the block's state transition meanwhile, so this is not +/// what the import waited: that is `lean_block_import_phase_seconds`'s +/// `engine` phase, zero whenever the answer was in before the transition +/// finished. This is the execution client's own processing time, observed for +/// pre-gloas blocks only (a gloas payload is asked about with its envelope). +/// `outcome` is `valid`, `optimistic`, `invalidated` or `no_verdict` (no +/// answer after the whole ladder, so the time is the ladder's). +pub fn observe_engine_new_payload(outcome: &str, duration: Duration) { + static LEAN_ENGINE_NEW_PAYLOAD_SECONDS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_histogram_vec!( + "lean_engine_new_payload_seconds", + "Time the execution client took to answer engine_newPayload for a block, \ + retries included", + &["outcome"], + // Up to the retry ladder's worst case: three attempts at the + // per-attempt timeout plus the backoff between them. + vec![ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, + 8.0, 12.0, 16.0, 32.0, + ] + ) + .unwrap() + }); + LEAN_ENGINE_NEW_PAYLOAD_SECONDS + .with_label_values(&[outcome]) + .observe(duration.as_secs_f64()); +} + /// Blocks not imported because the execution client has not validated them and /// they do not qualify for an optimistic import. /// diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index d5d1502df..73cd135df 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -121,9 +121,13 @@ keeps it harmless: exhausted retry ladder. The overlap shows in the import timing tree: the transition that ran during the -round trip is reported under `stf`, and `engine` reports only the wait that was -left once it finished. Without an execution client, and for forks that have no -request to send (pre-merge, gloas), nothing overlaps and the path is the old one. +round trip is reported under `stf` (used or discarded), and `engine` reports only +the wait that was left once it finished, a recorded zero when the answer was +already in. Without an execution client, and for forks that have no request to +send (pre-merge, gloas), nothing overlaps, `engine` is zero, and the path is the +old one. How long the execution client itself took to answer is +`lean_engine_new_payload_seconds{outcome}`, timed around the request in +`ask_request`. ## The verdicts diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 088faa3ad..0734caf1b 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -279,16 +279,19 @@ faster than the writer persists them, the wait shows up here rather than in With an execution client the beacon `stf` row also includes a state transition that ran while `engine_newPayload` was in flight, and `engine` then -reports only the wait left after it. A replay has no client, so neither applies -to its numbers. +reports only the wait left after it, zero when the answer was already in. The +execution client's own time is `lean_engine_new_payload_seconds`. A replay has +no client, so none of this applies to its numbers. ### What is excluded, and why Replay runs nothing that a corpus already answers for: -- **No execution client**, so `engine` reports a near-zero section with nothing - to call and `fcu` never runs. On a live follower paired with one, `engine` was - 31ms p50, about 0.5% of an import; a replay's numbers are complete without it. +- **No execution client**, so `engine` reports a zero section with nothing to + call and `fcu` never runs. On a live follower paired with one, the + `engine_newPayload` round trip was 31ms p50, about 0.5% of an import, before + the state transition was overlapped with it; a replay's numbers are complete + without it. - **An empty custody set**, so `columns_wait` never runs and no block is ever held waiting on data availability. A corpus supplies every block directly; there are no columns to wait for. diff --git a/docs/metrics.md b/docs/metrics.md index 441abd8c5..357e38770 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -253,7 +253,7 @@ The bookkeeping an import triggers (chain-event emission, the finality eviction On the beacon wire, `decode` for `source="gossip"` spans more than its name says. `BlockArrival::decode_start` is still the wire arrival, but `handed_off` is stamped only once the block has a gossip verdict, so the section also covers the cheap, stateless checks, the stateful check's own `spawn_blocking` task, and the verdict's trip back through the p2p actor's mailbox. There is no wait for a free validation slot to attribute here either: `try_acquire_owned` never blocks, and a message arriving with none free is reported `Ignore(Overloaded)` (and still forwarded to the chain actor) rather than queued. A rising `decode` on the beacon wire alone therefore does not mean decoding got slower; check `lean_beacon_gossip_validation_seconds` before assuming so. -`engine` and `fcu` are execution-client round trips. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. +`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` round trip the import could not hide behind its own state transition, so it is zero for most blocks; the round trip itself is `lean_engine_new_payload_seconds`. The `source` label has two values, `gossip` and `sync` (req/resp backfill). Two populations are deliberately absent. A block re-delivered to itself because its slot had not started reports under the source it first arrived on, since the hold is already visible as its `defer` section and a third label value would take the block out of the population it belongs to for every section it has left. A block this node built itself is not measured at all: it crossed no wire, so it has no `decode` and an empty `queue` taken when the import began. Both still appear in the log, which names them `deferred` and `local`. @@ -263,6 +263,7 @@ Series appear as blocks arrive rather than being seeded, since a third of the ph |------|------|-------|-------------------------|--------|---------| | `lean_block_import_phase_seconds` | Histogram | Time one section of a block's import, or of the arrival carrying it, took | Per section that ran, on each import, hold or failure | phase (see below); source=gossip,sync | 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | +| `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload` for a pre-gloas block, from the request to the answer, retry ladder included. The import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per beacon block asked of an execution client | outcome=valid,optimistic,invalidated,no_verdict | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_block_import_cascade_blocks` | Histogram | Blocks one arrival put through the import path (attempts, so a block that ends held or pended counts) | Once per arriving block message | | 1, 2, 3, 5, 8, 16, 32, 64, 128 | Per-block phases, in the order a block crosses them, plus `total` for a completed import: @@ -278,10 +279,10 @@ Per-block phases, in the order a block crosses them, plus `total` for a complete | `cascade_wait` | Between the parent's import finishing and this block being popped off the cascade queue | both | | `da_check` | The custody-column availability check itself, not the wait | beacon | | `columns_wait` | Held because custody columns had not all arrived | beacon | -| `engine` | The `engine_newPayload` round trip, including its retry ladder. When the state transition ran meanwhile and was used, only the wait left after it finished | beacon | +| `engine` | The wait for the `engine_newPayload` verdict left once the state transition that ran meanwhile finished. Observed as `0` for every beacon block whose answer was already in by then, and for every block with no request to send (no execution client, pre-merge, gloas). Non-zero only when the execution client was slower than the transition, or gave no answer (the rest of its retry ladder, block held) | beacon | | `verify_struct` | Participant bounds checks and pubkey resolution | lean | | `verify_crypto` | The leanVM multi-message aggregate verification | lean | -| `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it. It includes a transition that ran during the `engine` round trip | both | +| `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it. It includes the transition that ran during the `engine_newPayload` round trip, also when that one was then discarded and run again (an `INVALIDATED` verdict) or the block was held | both | | `writer_wait` | Beacon only: the importer's blocking hand-off of the post-state to the storage crate's background writer, which waits while the writer's queue is full. Subtracted from `stf`, so `stf` and `writer_wait` do not overlap | beacon | | `db_write` | The block write and handing the post-state off to the storage crate's background writer; the state's own encode/diff/commit cost is `lean_state_write_seconds` instead, not this row | lean | | `fc_head` | `update_head` | lean | From 088dfc2df293bfa00352d7f7789533a885923630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:12:31 -0300 Subject: [PATCH 03/11] perf(blockchain): time a gloas envelope's newPayload with the same metric lean_engine_new_payload_seconds covered only a pre-gloas block's payload, so from gloas on, when the payload is revealed in an envelope and asked about with engine_newPayloadV5, the execution client's time went unmeasured. Every newPayload request now goes through one helper, timed_new_payload, which observes the time from sending it to its answer under the verdict it reads as: the block path (ask_request, and ask) and the envelope path (ask_envelope_status, which ask_envelope also goes through) alike. An envelope refused before a request is sent, because it does not match its block, is not observed. --- crates/blockchain/src/beacon_engine.rs | 87 +++++++++++++++----------- crates/blockchain/src/lib.rs | 4 +- crates/blockchain/src/metrics.rs | 25 ++++---- docs/beacon_engine.md | 4 +- docs/metrics.md | 2 +- 5 files changed, 67 insertions(+), 55 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 14884b293..9707eb5f2 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -213,38 +213,51 @@ impl NewPayloadRequest<'_> { /// The answer is read exactly as [`ask`] reads it: `Err` means no answer after /// the client's whole retry ladder, and there is no `Ok(None)` because a /// request existing means there was something to ask. -/// -/// The time from sending the request to the answer is observed as -/// `lean_engine_new_payload_seconds`, under the answer's [`outcome_label`]. pub async fn ask_request( client: &EngineClient, request: OwnedNewPayloadRequest, ) -> Result { + let call = client.new_payload( + &request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ); + let status = timed_new_payload(call).await?; + Ok(verdict(&status)) +} + +/// Awaits `call`, one `engine_newPayload*` request, and observes the time from +/// sending it to its answer as `lean_engine_new_payload_seconds`, under the +/// [`outcome_label`] of the verdict the answer reads as. +/// +/// Every request this module sends goes through here, a block's payload and an +/// envelope's alike, so the metric is the execution client's time for every +/// payload it was asked about. A request refused before it is sent (an +/// envelope that does not match its block) never reaches it. +async fn timed_new_payload( + call: impl Future>, +) -> Result { let sent = Instant::now(); - let answer = client - .new_payload( - &request.execution_payload, - &request.versioned_hashes, - request.parent_beacon_block_root, - &request.execution_requests, - ) - .await - .map(|status| verdict(&status)); - crate::metrics::observe_engine_new_payload(outcome_label(&answer), sent.elapsed()); - answer + let status = call.await; + let validity = status.as_ref().ok().map(verdict); + let outcome = outcome_label(validity.as_ref()); + crate::metrics::observe_engine_new_payload(outcome, sent.elapsed()); + status } -/// The `outcome` an answer from [`ask_request`] is counted under. +/// The `outcome` an answer is counted under: its verdict, or `no_verdict` +/// when the retry ladder ran out without one. /// /// `not_required` is listed for completeness: a status from the execution /// client always reads as one of the other three verdicts. -pub fn outcome_label(answer: &Result) -> &'static str { - match answer { - Ok(PayloadValidity::Validated) => "valid", - Ok(PayloadValidity::Optimistic) => "optimistic", - Ok(PayloadValidity::Invalidated { .. }) => "invalidated", - Ok(PayloadValidity::NotRequired) => "not_required", - Err(_) => "no_verdict", +pub fn outcome_label(verdict: Option<&PayloadValidity>) -> &'static str { + match verdict { + Some(PayloadValidity::Validated) => "valid", + Some(PayloadValidity::Optimistic) => "optimistic", + Some(PayloadValidity::Invalidated { .. }) => "invalidated", + Some(PayloadValidity::NotRequired) => "not_required", + None => "no_verdict", } } @@ -262,14 +275,13 @@ pub async fn ask( let Some(request) = new_payload_request(block) else { return Ok(None); }; - let status = client - .new_payload( - request.execution_payload, - &request.versioned_hashes, - request.parent_beacon_block_root, - &request.execution_requests, - ) - .await?; + let call = client.new_payload( + request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ); + let status = timed_new_payload(call).await?; Ok(Some(verdict(&status))) } @@ -300,14 +312,13 @@ pub async fn ask_envelope_status( envelope: &containers::gloas::SignedExecutionPayloadEnvelope, ) -> Result { let request = gloas_new_payload_request(block, envelope)?; - client - .new_payload_v5( - request.execution_payload, - &request.versioned_hashes, - request.parent_beacon_block_root, - &request.execution_requests, - ) - .await + let call = client.new_payload_v5( + request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ); + timed_new_payload(call).await } #[cfg(test)] diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 1bfd4a064..446c8610d 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1812,8 +1812,8 @@ impl BlockChainServer { // nothing left to do, so it starts as a recorded zero and only // a request still unanswered when the transition is done // widens it. The execution client's own time is - // `lean_engine_new_payload_seconds`, observed in - // `ask_request`. + // `lean_engine_new_payload_seconds`, observed around the + // request itself. let no_wait = Instant::now(); (timings.engine_start, timings.engine_end) = (Some(no_wait), Some(no_wait)); let parent_root = beacon_block.parent_root(); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index e801ee6b2..c9ac2c754 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1534,24 +1534,25 @@ pub fn inc_engine_speculation(outcome: &str) { .inc(); } -/// How long the execution client took to answer `engine_newPayload` for a -/// block's payload, from sending the request to its answer, retry ladder -/// included. +/// How long the execution client took to answer `engine_newPayload*` for a +/// payload, from sending the request to its answer, retry ladder included: +/// a pre-gloas block's payload, and from gloas on the payload an envelope +/// reveals. /// -/// The import computes the block's state transition meanwhile, so this is not -/// what the import waited: that is `lean_block_import_phase_seconds`'s -/// `engine` phase, zero whenever the answer was in before the transition -/// finished. This is the execution client's own processing time, observed for -/// pre-gloas blocks only (a gloas payload is asked about with its envelope). -/// `outcome` is `valid`, `optimistic`, `invalidated` or `no_verdict` (no -/// answer after the whole ladder, so the time is the ladder's). +/// For a pre-gloas block the import computes the state transition meanwhile, +/// so this is not what the import waited: that is +/// `lean_block_import_phase_seconds`'s `engine` phase, zero whenever the +/// answer was in before the transition finished. This is the execution +/// client's own processing time. `outcome` is `valid`, `optimistic`, +/// `invalidated` or `no_verdict` (no answer after the whole ladder, so the +/// time is the ladder's). pub fn observe_engine_new_payload(outcome: &str, duration: Duration) { static LEAN_ENGINE_NEW_PAYLOAD_SECONDS: std::sync::LazyLock = std::sync::LazyLock::new(|| { register_histogram_vec!( "lean_engine_new_payload_seconds", - "Time the execution client took to answer engine_newPayload for a block, \ - retries included", + "Time the execution client took to answer engine_newPayload for a block's \ + or an envelope's payload, retries included", &["outcome"], // Up to the retry ladder's worst case: three attempts at the // per-attempt timeout plus the backoff between them. diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 73cd135df..621fd34b4 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -126,8 +126,8 @@ the wait that was left once it finished, a recorded zero when the answer was already in. Without an execution client, and for forks that have no request to send (pre-merge, gloas), nothing overlaps, `engine` is zero, and the path is the old one. How long the execution client itself took to answer is -`lean_engine_new_payload_seconds{outcome}`, timed around the request in -`ask_request`. +`lean_engine_new_payload_seconds{outcome}`, timed around every `engine_newPayload*` +request this module sends (`timed_new_payload`), a gloas envelope's included. ## The verdicts diff --git a/docs/metrics.md b/docs/metrics.md index 357e38770..cea7a6a90 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -263,7 +263,7 @@ Series appear as blocks arrive rather than being seeded, since a third of the ph |------|------|-------|-------------------------|--------|---------| | `lean_block_import_phase_seconds` | Histogram | Time one section of a block's import, or of the arrival carrying it, took | Per section that ran, on each import, hold or failure | phase (see below); source=gossip,sync | 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | -| `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload` for a pre-gloas block, from the request to the answer, retry ladder included. The import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per beacon block asked of an execution client | outcome=valid,optimistic,invalidated,no_verdict | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | +| `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload*`, from the request to the answer, retry ladder included: a pre-gloas block's payload, or from gloas on the payload an envelope reveals. For a block the import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per request sent: each pre-gloas beacon block, and each gloas envelope that passed its consensus checks | outcome=valid,optimistic,invalidated,no_verdict | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_block_import_cascade_blocks` | Histogram | Blocks one arrival put through the import path (attempts, so a block that ends held or pended counts) | Once per arriving block message | | 1, 2, 3, 5, 8, 16, 32, 64, 128 | Per-block phases, in the order a block crosses them, plus `total` for a completed import: From 257ae542552cd02aefd35b61d9e2dd52aa36b5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:11:21 -0300 Subject: [PATCH 04/11] refactor(blockchain): drop the dead engine asks and name the speculative timing fields Review cleanups on the engine/STF overlap: - remove beacon_engine::ask and ask_envelope, which no caller uses since process_block sends its request through ask_request, and fix the doc links and the closed-port test comment that still named them - add PayloadValidity::is_invalidated for the two matches! that tested it - rename ImportTimings::stf_spec_{start,end} to stf_speculative_*, since "spec" reads as "specification" here - say that a no_verdict newPayload sample also covers an Rpc error answer, which is not retried, so its time is not always the whole ladder's - docs/metrics.md: engine is zero only when the execution client answers before the transition finishes, not "for most blocks" - precomputed_epoch_state and lean_beacon_epoch_precompute_lookups_total count per transition run, not per import, now that an import can run a block's transition twice --- crates/blockchain/src/beacon_engine.rs | 64 +++++-------------- crates/blockchain/src/import_timing.rs | 14 ++-- crates/blockchain/src/lib.rs | 13 ++-- crates/blockchain/src/metrics.rs | 5 +- .../src/beacon/fork_choice.rs | 21 ++++-- .../state_transition/src/metrics.rs | 10 ++- docs/metrics.md | 6 +- 7 files changed, 59 insertions(+), 74 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 9707eb5f2..cc9699b83 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -207,12 +207,16 @@ impl NewPayloadRequest<'_> { } } -/// [`ask`] for a request already built with [`new_payload_request`], which is -/// how the actor sends it from a task while it computes the transition. +/// Asks the execution client about a block's payload, with a request already +/// built by [`new_payload_request`], which is how the actor sends it from a +/// task while it computes the transition. /// -/// The answer is read exactly as [`ask`] reads it: `Err` means no answer after -/// the client's whole retry ladder, and there is no `Ok(None)` because a -/// request existing means there was something to ask. +/// `Err` means no verdict: no answer after the client's whole retry ladder, or +/// a JSON-RPC error answer ([`EngineError::Rpc`]), which is not retried. +/// `optimistic-sync.md` then requires the caller not to import the block and +/// not to touch fork choice. There is no "nothing to ask" reading: a request +/// existing means there was something to ask, and a block with none (pre-merge, +/// gloas) is [`PayloadValidity::NotRequired`] without a call. pub async fn ask_request( client: &EngineClient, request: OwnedNewPayloadRequest, @@ -247,7 +251,9 @@ async fn timed_new_payload( } /// The `outcome` an answer is counted under: its verdict, or `no_verdict` -/// when the retry ladder ran out without one. +/// when there is none. That is either the retry ladder running out without an +/// answer, or a JSON-RPC error answer ([`EngineError::Rpc`]), which ends the +/// call at its first attempt since asking again would get the same refusal. /// /// `not_required` is listed for completeness: a status from the execution /// client always reads as one of the other three verdicts. @@ -261,51 +267,15 @@ pub fn outcome_label(verdict: Option<&PayloadValidity>) -> &'static str { } } -/// Asks the execution client about `block`'s payload. -/// -/// `Ok(None)` means there was nothing to ask, which is not a failure: -/// [`PayloadValidity::NotRequired`] is the right verdict and the block imports. -/// `Err` means no answer was obtained after the client's whole retry ladder, and -/// `optimistic-sync.md` requires the caller not to import the block and not to -/// touch fork choice. -pub async fn ask( - client: &EngineClient, - block: &SignedBeaconBlock, -) -> Result, EngineError> { - let Some(request) = new_payload_request(block) else { - return Ok(None); - }; - let call = client.new_payload( - request.execution_payload, - &request.versioned_hashes, - request.parent_beacon_block_root, - &request.execution_requests, - ); - let status = timed_new_payload(call).await?; - Ok(Some(verdict(&status))) -} - /// Asks the execution client about the payload `envelope` reveals for `block`, -/// with `engine_newPayloadV5`. +/// with `engine_newPayloadV5`, and hands back its raw answer. /// -/// Never `Ok(None)`: unlike [`ask`] there is no "nothing to ask" reading, and a -/// block/envelope pair that does not match is an -/// [`EngineError::EnvelopeMismatch`], which the caller must treat as a refusal -/// to import. -pub async fn ask_envelope( - client: &EngineClient, - block: &SignedBeaconBlock, - envelope: &containers::gloas::SignedExecutionPayloadEnvelope, -) -> Result { - ask_envelope_status(client, block, envelope) - .await - .map(|status| verdict(&status)) -} - -/// [`ask_envelope`] with the engine's raw answer, for a caller that must tell +/// Raw rather than read through [`verdict`], since the caller must tell /// `INVALID_BLOCK_HASH` (the envelope's contents do not hash to its claimed /// block hash, so only that envelope is bad) from `INVALID` (the payload the -/// hash names is bad). +/// hash names is bad). Never "nothing to ask": a block/envelope pair that does +/// not match is an [`EngineError::EnvelopeMismatch`], which the caller must +/// treat as a refusal to import. pub async fn ask_envelope_status( client: &EngineClient, block: &SignedBeaconBlock, diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index e3b0daba8..645f21dce 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -181,7 +181,7 @@ pub struct ImportTimings { /// work. /// /// The request is sent from a task of its own while the state transition - /// runs (see [`Self::stf_spec_start`]), so this pair starts where that + /// runs (see [`Self::stf_speculative_start`]), so this pair starts where that /// transition ended and ends when the answer arrived. Both are the same /// instant, a recorded zero, when there was no request to send (no /// execution client, or a block with no payload in it: pre-merge or @@ -220,8 +220,8 @@ pub struct ImportTimings { /// instead of counting the overlap twice. `None` whenever no request was /// sent, which is every import without an execution client and every /// block with no payload to ask about. - pub stf_spec_start: Option, - pub stf_spec_end: Option, + pub stf_speculative_start: Option, + pub stf_speculative_end: Option, /// Beacon: the blocking hand-off of the post-state to the storage crate's /// background writer, which waits while the writer's queue is full. It @@ -404,9 +404,9 @@ impl ImportTimings { } // Also when the import stopped before running its own transition (a // held block), so the speculation it paid for is not lost. - if let Some(spec) = span(self.stf_spec_start, self.stf_spec_end) { + if let Some(spec) = span(self.stf_speculative_start, self.stf_speculative_end) { stf.elapsed = Some(stf.elapsed.unwrap_or_default() + spec); - stf.end = stf.end.or(self.stf_spec_end); + stf.end = stf.end.or(self.stf_speculative_end); } vec![ Row::new("decode", self.decode_start, self.decode_end), @@ -936,7 +936,7 @@ mod tests { // engine row keeps the 30 ms that was left and stf gains the 70 ms. let base = Instant::now(); let mut timings = ImportTimings::default(); - (timings.stf_spec_start, timings.stf_spec_end) = pair(base, 0, 70); + (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 0, 70); (timings.engine_start, timings.engine_end) = pair(base, 70, 30); (timings.stf_start, timings.stf_end) = pair(base, 100, 20); @@ -966,7 +966,7 @@ mod tests { // the block was held without a transition of its own. let base = Instant::now(); let mut timings = ImportTimings::default(); - (timings.stf_spec_start, timings.stf_spec_end) = pair(base, 0, 70); + (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 0, 70); (timings.engine_start, timings.engine_end) = pair(base, 70, 300); assert_eq!( diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 446c8610d..85bfc41e2 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1845,8 +1845,8 @@ impl BlockChainServer { // import ends up using it: it is // state-transition work either way. if computed.is_some() { - timings.stf_spec_start = Some(spec_start); - timings.stf_spec_end = Some(spec_end); + timings.stf_speculative_start = Some(spec_start); + timings.stf_speculative_end = Some(spec_end); } speculation = computed; // A panicked or cancelled task is no @@ -1929,10 +1929,7 @@ impl BlockChainServer { if !spec.is_for(block_root, parent_root) { metrics::inc_engine_speculation("discarded_mismatch"); None - } else if matches!( - validity, - fork_choice::PayloadValidity::Invalidated { .. } - ) { + } else if validity.is_invalidated() { metrics::inc_engine_speculation("discarded_invalidated"); None } else { @@ -5741,8 +5738,8 @@ mod tests { /// /// Driven through the real engine client against a closed port, since an /// unreachable execution client is exactly the production shape of this: - /// `beacon_engine::ask` spends its retry ladder and `process_block` - /// answers `Held` with nothing recorded. + /// `beacon_engine::ask_request` spends its retry ladder and + /// `process_block` answers `Held` with nothing recorded. #[tokio::test] async fn a_release_that_gets_no_engine_verdict_puts_the_block_back_on_hold() { let store = beacon_store(GENESIS_TIME, 0); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index c9ac2c754..f77279d1c 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1544,8 +1544,9 @@ pub fn inc_engine_speculation(outcome: &str) { /// `lean_block_import_phase_seconds`'s `engine` phase, zero whenever the /// answer was in before the transition finished. This is the execution /// client's own processing time. `outcome` is `valid`, `optimistic`, -/// `invalidated` or `no_verdict` (no answer after the whole ladder, so the -/// time is the ladder's). +/// `invalidated` or `no_verdict`. A `no_verdict` sample is either the whole +/// retry ladder, when no answer came, or a single attempt, when the execution +/// client answered with a JSON-RPC error, which is not retried. pub fn observe_engine_new_payload(outcome: &str, duration: Duration) { static LEAN_ENGINE_NEW_PAYLOAD_SECONDS: std::sync::LazyLock = std::sync::LazyLock::new(|| { diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 2899d1ecb..f9915f068 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -664,6 +664,14 @@ pub enum PayloadValidity { }, } +impl PayloadValidity { + /// Whether this is [`PayloadValidity::Invalidated`], the one verdict under + /// which [`on_block`] runs the transition with an `invalid()` engine. + pub fn is_invalidated(&self) -> bool { + matches!(self, Self::Invalidated { .. }) + } +} + /// Reads an execution client's status as the verdict [`on_block`] takes. /// /// One function for both sources of a status: the real client's JSON-RPC answer @@ -1141,8 +1149,14 @@ fn transition_block( /// is before the first slot of the block's epoch, and the block is at or after /// that slot. The precompute worker stores it under the key /// [`checkpoint_state`] uses for `(block_epoch, parent_root)`, since the two -/// name the same state. Counts a hit or a miss for every such import and for -/// no other. +/// name the same state. +/// +/// Counts a hit or a miss each time a transition of such a block runs, which is +/// not always once per import: a speculation the chain actor computed while the +/// execution client was asked (see [`speculate_transition`]) counts even when it +/// is discarded, so an `INVALIDATED` epoch-crossing block, whose transition +/// runs again under `invalid()`, counts twice, and so does a used speculation in +/// a build with debug assertions, which re-runs the transition to check it. fn precomputed_epoch_state( store: &Store, parent_slot: Slot, @@ -5087,8 +5101,7 @@ pub fn on_block_speculative( // epoch; see `ActiveBalanceCacheExt`. let active_balances = store.active_balance_cache(); let usable = speculation.filter(|speculation| { - speculation.is_for(block_root, parent_root) - && !matches!(payload_validity, PayloadValidity::Invalidated { .. }) + speculation.is_for(block_root, parent_root) && !payload_validity.is_invalidated() }); let (mut state, transition) = match usable { Some(speculation) => (speculation.state, speculation.outcome), diff --git a/crates/blockchain/state_transition/src/metrics.rs b/crates/blockchain/state_transition/src/metrics.rs index 63c029166..8b61ca417 100644 --- a/crates/blockchain/state_transition/src/metrics.rs +++ b/crates/blockchain/state_transition/src/metrics.rs @@ -117,14 +117,18 @@ pub fn time_justified_balances_build() -> TimingGuard { static LEAN_BEACON_EPOCH_PRECOMPUTE_LOOKUPS_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "lean_beacon_epoch_precompute_lookups_total", - "Epoch-crossing block imports, by whether a precomputed epoch-boundary state was cached", + "Epoch-crossing block transitions, by whether a precomputed epoch-boundary state was cached", &["result"] ) .unwrap() }); -/// Count one epoch-crossing import's lookup of a precomputed state: `hit` or -/// `miss`. +/// Count one epoch-crossing transition's lookup of a precomputed state: `hit` +/// or `miss`. +/// +/// Per transition run, not per import: an import that runs a block's +/// transition twice (a discarded speculation, see +/// `fork_choice::precomputed_epoch_state`) looks the state up twice. pub fn inc_epoch_precompute_lookups(result: &str) { LEAN_BEACON_EPOCH_PRECOMPUTE_LOOKUPS_TOTAL .with_label_values(&[result]) diff --git a/docs/metrics.md b/docs/metrics.md index 59fe2be43..5cc6ce8f9 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -260,7 +260,7 @@ The bookkeeping an import triggers (chain-event emission, the finality eviction On the beacon wire, `decode` for `source="gossip"` spans more than its name says. `BlockArrival::decode_start` is still the wire arrival, but `handed_off` is stamped only once the block has a gossip verdict, so the section also covers the cheap, stateless checks, the stateful check's own `spawn_blocking` task, and the verdict's trip back through the p2p actor's mailbox. There is no wait for a free validation slot to attribute here either: `try_acquire_owned` never blocks, and a message arriving with none free is reported `Ignore(Overloaded)` (and still forwarded to the chain actor) rather than queued. A rising `decode` on the beacon wire alone therefore does not mean decoding got slower; check `lean_beacon_gossip_validation_seconds` before assuming so. -`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` round trip the import could not hide behind its own state transition, so it is zero for most blocks; the round trip itself is `lean_engine_new_payload_seconds`. +`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` round trip the import could not hide behind its own state transition: zero for a block whose answer arrived before that transition finished, the remainder of the round trip for one whose answer did not. How often it is zero depends on the execution client being faster than the transition. The round trip itself is `lean_engine_new_payload_seconds`. The `source` label has two values, `gossip` and `sync` (req/resp backfill). Two populations are deliberately absent. A block re-delivered to itself because its slot had not started reports under the source it first arrived on, since the hold is already visible as its `defer` section and a third label value would take the block out of the population it belongs to for every section it has left. A block this node built itself is not measured at all: it crossed no wire, so it has no `decode` and an empty `queue` taken when the import began. Both still appear in the log, which names them `deferred` and `local`. @@ -270,7 +270,7 @@ Series appear as blocks arrive rather than being seeded, since a third of the ph |------|------|-------|-------------------------|--------|---------| | `lean_block_import_phase_seconds` | Histogram | Time one section of a block's import, or of the arrival carrying it, took | Per section that ran, on each import, hold or failure | phase (see below); source=gossip,sync | 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | -| `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload*`, from the request to the answer, retry ladder included: a pre-gloas block's payload, or from gloas on the payload an envelope reveals. For a block the import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per request sent: each pre-gloas beacon block, and each gloas envelope that passed its consensus checks | outcome=valid,optimistic,invalidated,no_verdict | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | +| `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload*`, from the request to the answer, retry ladder included: a pre-gloas block's payload, or from gloas on the payload an envelope reveals. For a block the import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per request sent: each pre-gloas beacon block, and each gloas envelope that passed its consensus checks | outcome=valid,optimistic,invalidated,no_verdict (`no_verdict`: the whole retry ladder with no answer, or one attempt answered with a JSON-RPC error, which is not retried) | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_block_import_cascade_blocks` | Histogram | Blocks one arrival put through the import path (attempts, so a block that ends held or pended counts) | Once per arriving block message | | 1, 2, 3, 5, 8, 16, 32, 64, 128 | Per-block phases, in the order a block crosses them, plus `total` for a completed import: @@ -557,7 +557,7 @@ leanMetrics spec. | Name | Type | Usage | Sample collection event | Labels | Buckets | |------|------|-------|-------------------------|--------|---------| -| `lean_beacon_epoch_precompute_lookups_total` | Counter | Epoch-crossing imports, by whether a precomputed boundary state was cached | On each import whose parent is before the block's epoch start and whose block is at or after it | result=hit,miss | | +| `lean_beacon_epoch_precompute_lookups_total` | Counter | Epoch-crossing block transitions, by whether a precomputed boundary state was cached | On each state transition of a block whose parent is before the block's epoch start and whose block is at or after it. Usually once per import; twice when the import runs the transition again (a speculation computed during the `engine_newPayload` round trip and then discarded for an `INVALIDATED` verdict, or re-checked in a debug build), and once for a block held after its speculation ran | result=hit,miss | | | `lean_beacon_epoch_precompute_seconds` | Histogram | Worker time to advance, flush and hash one state | When a precompute worker finishes | | 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 8 | | `lean_beacon_epoch_precompute_started_total` | Counter | Workers started, by trigger | When a worker is spawned | trigger=head,timer | | From 2f44d99256495fdb2a7042d6fafbaf5ba4a16b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:31:06 -0300 Subject: [PATCH 05/11] fix(fork_choice): check a used speculation in debug builds and never panic computing one Two review points on the speculative transition: - The speculation is sound because the chain actor handles one message at a time and every cache transition_block reads is content-keyed. That was only documented. With debug assertions on (every test build, release-fast included), on_block_speculative now also runs the transition whenever it uses a speculation and panics unless both agree on the outcome and, for a success, on the post-state root. The poisoned-speculation test becomes the check that it fires, and a new one covers the post-state arm. - speculate_transition turned a storage read error into a panic on the actor's thread. It is optional work, so it now answers None and leaves the error to on_block's own lookup. Also corrects "without touching the store": the transition can cache an epoch-crossing block's boundary balances, a content-keyed entry that is right whatever becomes of the block. --- .../src/beacon/fork_choice.rs | 122 ++++++++++++++++-- 1 file changed, 112 insertions(+), 10 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index f9915f068..378958ff9 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4832,8 +4832,12 @@ pub fn on_block( /// /// Opaque on purpose: the only thing a holder can do with one is hand it to /// [`on_block_speculative`], which decides whether it may stand in for the -/// transition `on_block` would otherwise run. Until then it is a local value; -/// nothing here has reached the store, and dropping it costs only the work. +/// transition `on_block` would otherwise run. Until then it is a local value: +/// the block, its post-state and its fork choice records have not reached the +/// store, and dropping it costs only the work. Computing it may still write one +/// content-keyed cache entry, the boundary balances of an epoch-crossing block's +/// checkpoint (see [`speculate_transition`]), which is the entry `on_block`'s +/// own transition would write and is correct whether or not the block imports. /// /// # Why it is the transition `on_block` would run /// @@ -4853,6 +4857,12 @@ pub fn on_block( /// block, the config and the committee and active-balance caches, which are /// content-keyed as well. The chain actor handles one message at a time, so /// nothing mutates the store between this computation and `on_block`'s own. +/// +/// That last sentence is an argument, not something the types enforce, so +/// builds with debug assertions (the test suite's `release-fast` included) +/// check it: whenever [`on_block_speculative`] uses a speculation it also runs +/// the transition itself and panics unless both agree on the outcome and, for +/// a success, on the post-state root. pub struct SpeculativeTransition { block_root: Root, parent_root: Root, @@ -4868,15 +4878,49 @@ impl SpeculativeTransition { pub fn is_for(&self, block_root: Root, parent_root: Root) -> bool { self.block_root == block_root && self.parent_root == parent_root } + + /// Panics unless `fresh`, the transition `on_block` computed itself, agrees + /// with this speculation: the same outcome and, when it succeeded, the same + /// post-state root. + /// + /// Outcomes are compared through their `Debug` rendering, since the error + /// type carries no `PartialEq`, and roots only on success, since a failed + /// transition's partial state is never used. Debug builds only: it doubles + /// the transition, which is the cost the speculation exists to hide. + #[cfg(debug_assertions)] + fn assert_agrees_with(&self, fresh: &(BeaconState, Result<()>)) { + let (fresh_state, fresh_outcome) = fresh; + assert_eq!( + format!("{:?}", self.outcome), + format!("{fresh_outcome:?}"), + "the speculative transition disagrees with on_block's own on the outcome" + ); + if self.outcome.is_ok() { + assert_eq!( + self.state.hash_tree_root(), + fresh_state.hash_tree_root(), + "the speculative transition disagrees with on_block's own on the post-state root" + ); + } + } } /// Runs `signed_block`'s state transition against its parent's post-state with -/// the execution engine assumed valid, without touching the store. +/// the execution engine assumed valid, without importing anything. +/// +/// The store is only read, with one exception: an epoch-crossing block's +/// transition caches its boundary checkpoint's balances +/// ([`cache_boundary_balances`]) and counts a precomputed-state lookup, exactly +/// as `on_block`'s own transition would. The entry is keyed by the checkpoint +/// and built from the parent's state alone, so it is right whatever becomes of +/// the block. /// /// `None` when the parent has no post-state, the same condition under which /// `on_block` fails with `block.parent_root in store.block_states`; the caller -/// then gets that failure from `on_block` itself. The parent state is looked up -/// exactly as `on_block` looks it up. +/// then gets that failure from `on_block` itself. Also `None` when reading the +/// parent's state fails: this is optional work on the chain actor's thread, so +/// a storage error must not panic it here, and `on_block` meets the same error +/// on its own lookup and reports it there. /// /// None of `on_block`'s own checks run here. They run, unchanged and in the /// same order, when the speculation is handed to [`on_block_speculative`], so a @@ -4890,7 +4934,7 @@ pub fn speculate_transition( ) -> Option { let block_root = signed_block.message_hash_tree_root(); let parent_root = signed_block.parent_root(); - let parent_state = store.get_state(&parent_root).expect("get")?; + let parent_state = store.get_state(&parent_root).ok().flatten()?; let active_balances = store.active_balance_cache(); let (state, outcome) = transition_block( store, @@ -4920,6 +4964,9 @@ pub fn speculate_transition( /// disagree with), its result stands in for that call. In every other case it /// is dropped and the transition runs here as it always has, so the /// invalidation path in particular is the one `on_block` has today. +/// +/// With debug assertions on, a speculation that is used is also checked +/// against the transition run here (see [`SpeculativeTransition`]). pub fn on_block_speculative( store: &mut Store, signed_block: SignedBeaconBlock, @@ -5103,9 +5150,8 @@ pub fn on_block_speculative( let usable = speculation.filter(|speculation| { speculation.is_for(block_root, parent_root) && !payload_validity.is_invalidated() }); - let (mut state, transition) = match usable { - Some(speculation) => (speculation.state, speculation.outcome), - None => transition_block( + let fresh_transition = || { + transition_block( store, &parent_state, parent_root, @@ -5115,7 +5161,15 @@ pub fn on_block_speculative( &engine, committees, &active_balances, - ), + ) + }; + let (mut state, transition) = match usable { + Some(speculation) => { + #[cfg(debug_assertions)] + speculation.assert_agrees_with(&fresh_transition()); + (speculation.state, speculation.outcome) + } + None => fresh_transition(), }; // `optimistic-sync.md`: a block deemed `INVALIDATED` MUST NOT be included @@ -7524,7 +7578,18 @@ mod tests { Some(speculation) } + /// Without debug assertions the poisoned outcome is the import's result, + /// which proves the speculation stood in for the transition. With them + /// (every test build, `release-fast` included) `on_block_speculative` also + /// runs the transition and refuses to use a speculation that disagrees, so + /// the same import is the check that the invariant is enforced. #[test] + #[cfg_attr( + debug_assertions, + should_panic( + expected = "the speculative transition disagrees with on_block's own on the outcome" + ) + )] fn a_matching_speculation_is_the_transition_on_block_uses() { let (result, store, root) = import_valid_child(&PayloadValidity::Validated, poisoned); assert_eq!(failed_assertion(result), Some(POISON)); @@ -7534,6 +7599,43 @@ mod tests { ); } + /// The post-state half of the same check: a speculation that succeeded, as + /// the real transition does, but holds a different state. Without debug + /// assertions that state is what the import stores. + #[test] + #[cfg_attr( + debug_assertions, + should_panic( + expected = "the speculative transition disagrees with on_block's own on the post-state root" + ) + )] + fn a_speculation_with_another_post_state_is_caught_in_debug_builds() { + let (_, honest_store, root) = + import_valid_child(&PayloadValidity::Validated, honest_speculation); + let honest_root = honest_store + .get_state(&root) + .expect("get") + .expect("the honest import succeeds") + .hash_tree_root(); + let (result, store, _) = + import_valid_child(&PayloadValidity::Validated, |store, block, config| { + let mut speculation = honest_speculation(store, block, config)?; + speculation + .outcome + .as_ref() + .expect("the transition succeeds"); + speculation.state.latest_block_header_mut().body_root = Root::repeat_byte(0xab); + Some(speculation) + }); + result.expect("the import takes the speculation's outcome"); + let stored = store + .get_state(&root) + .expect("get") + .expect("imported") + .hash_tree_root(); + assert_ne!(stored, honest_root, "the speculation's state was stored"); + } + #[test] fn a_speculation_for_another_root_or_parent_is_ignored() { for tamper in [0u8, 1] { From e84a453a8c6cf43e38b47af75f8c5a4f83069cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:35:01 -0300 Subject: [PATCH 06/11] fix(blockchain): abort an abandoned engine request and keep a failed task apart from no verdict Three review points on process_block's overlapped engine request: - The spawned request task is held in tokio-util's AbortOnDropHandle, so an import dropped mid-await (a timeout or select! around the handler) no longer leaves a request running out its retry ladder with nobody reading the answer. Enables tokio-util's rt feature; Cargo.lock is unchanged. - The verdict's error is a two-variant NoVerdict instead of a String. An EngineError is the execution client's no-verdict, logged and counted in lean_engine_no_verdict_total as before. A JoinError (the task panicked or was cancelled) is this node's fault: logged at error and not counted there. Both still hold the block. - What became of the speculation is a SpeculationOutcome recorded on the pass's ImportTimings and in lean_engine_speculation_total from one helper, and logged on the import timing line, so a block whose stf row holds two transitions says so. The speculative marks are reset per pass, like the other import-pass marks. --- crates/blockchain/Cargo.toml | 2 +- crates/blockchain/src/import_timing.rs | 44 ++++++ crates/blockchain/src/lib.rs | 194 ++++++++++++++----------- docs/beacon_engine.md | 28 +++- docs/metrics.md | 2 +- 5 files changed, 179 insertions(+), 91 deletions(-) diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index f6aaa1136..cf7b0b8f6 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -26,7 +26,7 @@ libssz.workspace = true spawned-concurrency.workspace = true tokio.workspace = true -tokio-util = { version = "0.7", default-features = false } +tokio-util = { version = "0.7", default-features = false, features = ["rt"] } serde.workspace = true thiserror.workspace = true diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index 645f21dce..353237ea7 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -107,6 +107,37 @@ pub struct VerifyTimings { pub crypto_end: Option, } +/// What became of the state transition a beacon import computed while the +/// execution client's verdict was in flight (see +/// [`ImportTimings::stf_speculative_start`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpeculationOutcome { + /// Handed to `fork_choice::on_block_speculative` with a verdict that lets it + /// stand in for the transition. A block that then fails one of `on_block`'s + /// earlier checks still counts, since the work was spent either way. + Used, + /// The verdict was `INVALIDATED`, so `on_block` runs its own transition + /// with an `invalid()` engine. + DiscardedInvalidated, + /// The block was held without importing: no verdict, or a `SYNCING` or + /// `ACCEPTED` answer for a block that is not an optimistic candidate. + DiscardedHeld, + /// Computed for another root or parent. Not expected from the actor. + DiscardedMismatch, +} + +impl SpeculationOutcome { + /// The `outcome` label of `lean_engine_speculation_total`. + pub fn label(self) -> &'static str { + match self { + Self::Used => "used", + Self::DiscardedInvalidated => "discarded_invalidated", + Self::DiscardedHeld => "discarded_held", + Self::DiscardedMismatch => "discarded_mismatch", + } + } +} + /// Every boundary one block crosses on its way to a post-state. /// /// Carried alongside the block itself: through the actor mailbox in @@ -222,6 +253,11 @@ pub struct ImportTimings { /// block with no payload to ask about. pub stf_speculative_start: Option, pub stf_speculative_end: Option, + /// Beacon: what became of that speculative transition, set by + /// [`Self::record_speculation`]. Not a mark, like [`Self::source`]: it + /// travels with them so the import's log line can say whether its `stf` + /// row holds a transition that was discarded and run again. + pub speculation: Option, /// Beacon: the blocking hand-off of the post-state to the storage crate's /// background writer, which waits while the writer's queue is full. It @@ -258,6 +294,13 @@ pub struct ImportTimings { } impl ImportTimings { + /// Records what became of this pass's speculative transition, here and in + /// `lean_engine_speculation_total`, so the two cannot disagree. + pub fn record_speculation(&mut self, outcome: SpeculationOutcome) { + self.speculation = Some(outcome); + metrics::inc_engine_speculation(outcome.label()); + } + /// The `source` label this block reports under, or `None` for a block /// that is not reported at all. /// @@ -602,6 +645,7 @@ impl BlockImportReport { slot_offset_ms = self.slot_offset_ms, bottleneck = self.bottleneck(), da_complete_on_arrival = self.timings.da_complete_on_arrival, + speculation = self.timings.speculation.map(SpeculationOutcome::label), "Block import timing" ); for line in lines { diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 85bfc41e2..08c91e703 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1,4 +1,6 @@ -use ethlambda_engine::{CustodyColumns, EngineClient, PayloadStatusV1 as EnginePayloadStatus}; +use ethlambda_engine::{ + CustodyColumns, EngineClient, EngineError, PayloadStatusV1 as EnginePayloadStatus, +}; use ethlambda_network_api::{ AggregateArrival, BlockAnnouncement, BlockArrival, BlockChainToP2PRef, BlockSource, DeferredFrom, FetchRequest, InitP2P, @@ -42,13 +44,16 @@ use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Backend, Context, Handler, send_after, }; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, info, trace, warn}; use crate::beacon_columns::{ColumnParking, ParkedColumn}; use crate::beacon_envelope::EnvelopeQueues; use crate::block_builder::ProposerConfig; use crate::events::ChainEvents; -use crate::import_timing::{BlockImportReport, CascadeTimings, HeadTimings, ImportTimings}; +use crate::import_timing::{ + BlockImportReport, CascadeTimings, HeadTimings, ImportTimings, SpeculationOutcome, +}; use crate::store::StoreError; pub use events::{ChainEvent, EventBus, Topic, UnknownTopic}; @@ -765,6 +770,19 @@ pub enum ImportOutcome { Held, } +/// Why a beacon import has no verdict on its block's payload. Either way the +/// block is held, but only the first is the execution client's doing, so the +/// two are logged and counted apart. +enum NoVerdict { + /// No answer after the client's whole retry ladder, or a JSON-RPC error + /// answer. Counted in `lean_engine_no_verdict_total`. + Engine(EngineError), + /// The task that sent the request panicked or was cancelled. A fault in + /// this node, not in the execution client, so it is logged as an error + /// and not counted as the execution client's no-verdict. + Task(tokio::task::JoinError), +} + /// The availability evidence for `block`, or `None` if a column this node /// custodies has not arrived yet. /// @@ -1816,78 +1834,92 @@ impl BlockChainServer { // request itself. let no_wait = Instant::now(); (timings.engine_start, timings.engine_end) = (Some(no_wait), Some(no_wait)); + // Per pass, like every import-pass mark: a block held here and + // re-imported later must not report the earlier pass's + // speculation as this one's. + (timings.stf_speculative_start, timings.stf_speculative_end) = (None, None); + timings.speculation = None; let parent_root = beacon_block.parent_root(); let mut speculation: Option = None; - let verdict: Result, String> = match &self - .engine - { - None => Ok(None), - Some(client) => { - match beacon_engine::new_payload_request(&beacon_block) - .map(|request| request.into_owned()) - { - None => Ok(None), - Some(request) => { - let client = client.clone(); - let task = tokio::spawn(async move { - let answer = beacon_engine::ask_request(&client, request).await; - (answer, Instant::now()) - }); - let spec_start = Instant::now(); - let computed = fork_choice::speculate_transition( - &self.store, - &beacon_block, - &config, - &committees, - ); - let spec_end = Instant::now(); - // Charged to `stf` whether or not the - // import ends up using it: it is - // state-transition work either way. - if computed.is_some() { - timings.stf_speculative_start = Some(spec_start); - timings.stf_speculative_end = Some(spec_end); - } - speculation = computed; - // A panicked or cancelled task is no - // answer, like an exhausted retry ladder. - let (answer, answered_at) = match task.await { - Ok((answer, answered_at)) => ( - answer.map(Some).map_err(|err| err.to_string()), - answered_at, - ), - Err(err) => (Err(err.to_string()), Instant::now()), - }; - // The wait begins once the transition is - // done; an answer already in by then - // leaves it at zero. - timings.engine_start = Some(spec_end); - timings.engine_end = Some(answered_at.max(spec_end)); - answer - } + let request = self.engine.as_ref().and_then(|client| { + let request = beacon_engine::new_payload_request(&beacon_block)?; + Some((client.clone(), request.into_owned())) + }); + let verdict = match request { + None => Ok(fork_choice::PayloadValidity::NotRequired), + Some((client, request)) => { + // Aborted when dropped, so an import abandoned + // mid-await (a timeout or `select!` around this + // handler) does not leave the request running for + // the rest of its retry ladder with nobody to read + // the answer. + let task = AbortOnDropHandle::new(tokio::spawn(async move { + let answer = beacon_engine::ask_request(&client, request).await; + (answer, Instant::now()) + })); + let speculative_start = Instant::now(); + let computed = fork_choice::speculate_transition( + &self.store, + &beacon_block, + &config, + &committees, + ); + let speculative_end = Instant::now(); + // Charged to `stf` whether or not the import ends up + // using it: it is state-transition work either way. + if computed.is_some() { + timings.stf_speculative_start = Some(speculative_start); + timings.stf_speculative_end = Some(speculative_end); } + speculation = computed; + let (answer, answered_at) = match task.await { + Ok((answer, answered_at)) => { + (answer.map_err(NoVerdict::Engine), answered_at) + } + Err(err) => (Err(NoVerdict::Task(err)), Instant::now()), + }; + // The wait begins once the transition is done; an + // answer already in by then leaves it at zero. + timings.engine_start = Some(speculative_end); + timings.engine_end = Some(answered_at.max(speculative_end)); + answer } }; + // No verdict, whatever the reason. `optimistic-sync.md`: a + // consensus engine MUST NOT import the block and MUST NOT + // apply it to the fork choice store. Returning `Held` leaves + // the block out of the store without marking it imported, so + // the cascade does not treat it as having produced a + // post-state. let validity = match verdict { - Ok(None) => fork_choice::PayloadValidity::NotRequired, - Ok(Some(validity)) => validity, - // No answer after the whole ladder. - // `optimistic-sync.md`: a consensus engine MUST NOT - // import the block and MUST NOT apply it to the fork - // choice store. Returning `Held` leaves the block out - // of the store without marking it imported, so the - // cascade does not treat it as having produced a - // post-state. - Err(err) => { - warn!( - %slot, - block_root = %ShortRoot(&block_root.0), - %err, - "No verdict from the execution client; not importing" - ); - metrics::inc_engine_no_verdict(); + Ok(validity) => validity, + Err(no_verdict) => { + match no_verdict { + // The execution client's doing: no answer after + // the whole ladder, or a JSON-RPC error answer. + NoVerdict::Engine(err) => { + warn!( + %slot, + block_root = %ShortRoot(&block_root.0), + %err, + "No verdict from the execution client; not importing" + ); + metrics::inc_engine_no_verdict(); + } + // This node's: the task that sent the request + // panicked or was cancelled. Not counted as the + // execution client's no-verdict. + NoVerdict::Task(err) => { + error!( + %slot, + block_root = %ShortRoot(&block_root.0), + %err, + "The engine request task failed; not importing" + ); + } + } if speculation.is_some() { - metrics::inc_engine_speculation("discarded_held"); + timings.record_speculation(SpeculationOutcome::DiscardedHeld); } return (timings, Ok(ImportOutcome::Held)); } @@ -1913,7 +1945,7 @@ impl BlockChainServer { ); metrics::inc_engine_not_optimistic_candidate(); if speculation.is_some() { - metrics::inc_engine_speculation("discarded_held"); + timings.record_speculation(SpeculationOutcome::DiscardedHeld); } return (timings, Ok(ImportOutcome::Held)); } @@ -1924,21 +1956,17 @@ impl BlockChainServer { // run the transition itself with `invalid()`, exactly as // before. A discarded speculation stays in the `stf` row, so // the transition run twice shows as twice. - let speculation = match speculation { - Some(spec) => { - if !spec.is_for(block_root, parent_root) { - metrics::inc_engine_speculation("discarded_mismatch"); - None - } else if validity.is_invalidated() { - metrics::inc_engine_speculation("discarded_invalidated"); - None - } else { - metrics::inc_engine_speculation("used"); - Some(spec) - } - } - None => None, - }; + let speculation = speculation.and_then(|speculation| { + let outcome = if !speculation.is_for(block_root, parent_root) { + SpeculationOutcome::DiscardedMismatch + } else if validity.is_invalidated() { + SpeculationOutcome::DiscardedInvalidated + } else { + SpeculationOutcome::Used + }; + timings.record_speculation(outcome); + (outcome == SpeculationOutcome::Used).then_some(speculation) + }); // Drop any hand-off span left by an insert outside this call, // so the one taken below is this block's alone. diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 077ec6bf2..47cd6dbe3 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -106,10 +106,19 @@ and `valid()` for `NOT_REQUIRED`, `VALID` and optimistic alike, and the transition reads nothing else off the store except content-keyed caches. What keeps it harmless: -- The result is a local value (`SpeculativeTransition`). Nothing reaches the - store, a cache that is not content-addressed, or the network until - `on_block_speculative` accepts it, and that runs every check `on_block` makes, - unchanged and in the same order. +- The result is a local value (`SpeculativeTransition`). The block, its + post-state and its fork choice records reach neither the store nor the network + until `on_block_speculative` accepts it, and that runs every check `on_block` + makes, unchanged and in the same order. The one write computing it can make is + an epoch-crossing block's boundary balances, a cache entry keyed by the + checkpoint and built from the parent's state alone, so it is the entry + `on_block`'s own transition would write and is right whatever becomes of the + block. +- The argument above rests on the actor handling one message at a time, which + no type enforces. Builds with debug assertions (the test suite's + `release-fast` among them) therefore run the transition again whenever a + speculation is used and panic unless both agree on the outcome and the + post-state root. - It stands in for the transition only when it was computed for this block's root and parent root and the verdict is not `INVALIDATED`. For `INVALIDATED` it is dropped and `on_block` recomputes with `invalid()`, so the invalidation @@ -117,8 +126,15 @@ keeps it harmless: - A block that ends up held (no verdict, or `SYNCING`/`ACCEPTED` for a block that is not an optimistic candidate) drops it. The attempt is counted in `lean_engine_speculation_total{outcome}`. -- A panicked or cancelled request task is treated as no verdict, like an - exhausted retry ladder. +- A panicked or cancelled request task holds the block too, since there is no + verdict, but it is a fault in this node rather than in the execution client: + it is logged as an error and is not counted in `lean_engine_no_verdict_total`. +- The request task is aborted if the import is dropped before the answer is + read, so an abandoned import never leaves a request running out its retry + ladder. +- Reading the parent's state for the speculation never panics the actor: a + storage error there skips the speculation, and `on_block` meets the same error + on its own lookup. The overlap shows in the import timing tree: the transition that ran during the round trip is reported under `stf` (used or discarded), and `engine` reports only diff --git a/docs/metrics.md b/docs/metrics.md index 5cc6ce8f9..fa3614baa 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -260,7 +260,7 @@ The bookkeeping an import triggers (chain-event emission, the finality eviction On the beacon wire, `decode` for `source="gossip"` spans more than its name says. `BlockArrival::decode_start` is still the wire arrival, but `handed_off` is stamped only once the block has a gossip verdict, so the section also covers the cheap, stateless checks, the stateful check's own `spawn_blocking` task, and the verdict's trip back through the p2p actor's mailbox. There is no wait for a free validation slot to attribute here either: `try_acquire_owned` never blocks, and a message arriving with none free is reported `Ignore(Overloaded)` (and still forwarded to the chain actor) rather than queued. A rising `decode` on the beacon wire alone therefore does not mean decoding got slower; check `lean_beacon_gossip_validation_seconds` before assuming so. -`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` round trip the import could not hide behind its own state transition: zero for a block whose answer arrived before that transition finished, the remainder of the round trip for one whose answer did not. How often it is zero depends on the execution client being faster than the transition. The round trip itself is `lean_engine_new_payload_seconds`. +`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` round trip the import could not hide behind its own state transition: zero for a block whose answer arrived before that transition finished, the remainder of the round trip for one whose answer did not. How often it is zero depends on the execution client being faster than the transition. The round trip itself is `lean_engine_new_payload_seconds`. The transition that ran meanwhile is charged to `stf` whether or not the import used it, and the log line's `speculation` field says which (`used`, `discarded_invalidated`, `discarded_held` or `discarded_mismatch`), so a block whose `stf` holds two transitions says so. The `source` label has two values, `gossip` and `sync` (req/resp backfill). Two populations are deliberately absent. A block re-delivered to itself because its slot had not started reports under the source it first arrived on, since the hold is already visible as its `defer` section and a third label value would take the block out of the population it belongs to for every section it has left. A block this node built itself is not measured at all: it crossed no wire, so it has no `decode` and an empty `queue` taken when the import began. Both still appear in the log, which names them `deferred` and `local`. From 1290c37430ca4abf1a597b9dd8728fb33b69aea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:37:43 -0300 Subject: [PATCH 07/11] test(blockchain): drive the overlapped import against an answering execution client The only test that crossed process_block's speculation path was the closed-port one, which covers no verdict. Two more drive it against the mock execution client the envelope tests use: - INVALID: the speculation is recorded as discarded_invalidated, on_block gets past its guards and fails in the transition it runs itself, and nothing is imported. - SYNCING for a block that is not an optimistic candidate (genesis now, so the wall clock makes slot 1 recent and the premise is asserted): the block is held and the speculation recorded as discarded_held. Both assert the mock saw exactly one engine_newPayloadV4, so a verdict that never arrived (a retried, undecodable answer) cannot pass as held. The mock now answers every engine_newPayload* version with its newPayload status rather than only V5, and its helpers are pub(crate) for these tests. --- crates/blockchain/src/beacon_envelope.rs | 20 ++-- crates/blockchain/src/lib.rs | 125 +++++++++++++++++++++++ 2 files changed, 137 insertions(+), 8 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index a893a651c..4beac299f 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -1190,7 +1190,7 @@ impl BlockChainServer { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::path::PathBuf; use std::sync::Arc; @@ -2189,8 +2189,9 @@ mod tests { /// What the mock execution client answers and has been asked. #[derive(Default)] - struct MockState { - /// The `payloadStatus` object `engine_newPayloadV5` answers with. + pub(crate) struct MockState { + /// The `payloadStatus` object every `engine_newPayload*` answers with: + /// a gloas envelope's V5 and a pre-gloas block's V4 alike. new_payload: String, /// The `payloadStatus` object `engine_forkchoiceUpdated*` answers with. forkchoice: String, @@ -2198,14 +2199,17 @@ mod tests { requests: Vec, } - fn status_json(status: &str, latest_valid_hash: Option) -> String { + pub(crate) fn status_json(status: &str, latest_valid_hash: Option) -> String { let hash = latest_valid_hash.map_or("null".to_string(), |hash| format!("\"{hash:?}\"")); format!(r#"{{"status":"{status}","latestValidHash":{hash},"validationError":null}}"#) } /// A hand-rolled JSON-RPC server answering by method name, like /// `ethlambda-engine`'s wire smoke tests but for many requests. - async fn mock_engine( + /// + /// Shared with the block import tests in `lib.rs`, which ask V4 about a + /// fulu block's payload, so every `engine_newPayload*` gets `new_payload`. + pub(crate) async fn mock_engine( new_payload: String, forkchoice: String, ) -> (EngineClient, Arc>) { @@ -2262,13 +2266,13 @@ mod tests { let result = { let mut state = shared.lock().unwrap(); state.requests.push(body.clone()); - if body.contains("engine_newPayloadV5") { + if body.contains("engine_newPayload") { state.new_payload.clone() } else { state.forkchoice.clone() } }; - let result = if body.contains("engine_newPayloadV5") { + let result = if body.contains("engine_newPayload") { result } else { format!(r#"{{"payloadStatus":{result},"payloadId":null}}"#) @@ -2294,7 +2298,7 @@ mod tests { (client, state) } - fn methods(state: &Arc>) -> Vec { + pub(crate) fn methods(state: &Arc>) -> Vec { state .lock() .unwrap() diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 08c91e703..dfb082328 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -5824,6 +5824,131 @@ mod tests { ); } + // ----------------------------------------------------------------- + // The state transition overlapped with an answering execution client. + // + // `process_block` sends `engine_newPayloadV4` from a task and computes + // the block's transition meanwhile. These drive it against the mock + // execution client the envelope tests use, for the verdicts that must + // throw that speculation away. The block is the structurally minimal + // fulu one over `bare_state`, so its transition fails whatever the + // verdict; what is under test is the verdict's routing, not the import. + // ----------------------------------------------------------------- + + /// A server asking `client`, holding a post-state for a zero parent root, + /// and a payload-carrying fulu block at slot 1 on that parent. + fn server_asking( + client: EngineClient, + genesis_time: u64, + ) -> (BlockChainServer, SignedBeaconBlock) { + let mut server = beacon_server(beacon_store(genesis_time, 0)); + server.engine = Some(client); + // A block at the zero root too, for `on_block`'s finalized-chain walk. + server + .store + .insert_signed_block(H256::ZERO, bare_block(0, H256::repeat_byte(0xcc))) + .expect("insert"); + server + .store + .insert_state(H256::ZERO, bare_state()) + .expect("insert"); + (server, fulu_block(H256::ZERO, 1, 0)) + } + + fn new_payload_v4_requests( + mock: &Arc>, + ) -> usize { + crate::beacon_envelope::tests::methods(mock) + .iter() + .filter(|method| *method == "engine_newPayloadV4") + .count() + } + + /// `INVALID`: the speculation, computed with a `valid()` engine, is + /// dropped, `on_block` runs the transition itself with `invalid()`, and + /// nothing is imported. + #[tokio::test] + async fn an_invalid_verdict_discards_the_speculation_and_imports_nothing() { + use crate::beacon_envelope::tests::{mock_engine, status_json}; + + let (client, mock) = + mock_engine(status_json("INVALID", None), status_json("SYNCING", None)).await; + let (mut server, block) = server_asking(client, GENESIS_TIME); + let block_root = block.message_hash_tree_root(); + // The store's clock at the block's slot, so `on_block` gets past its + // future-block guard and reaches the verdict. + let config = server.store.config(); + fork_choice::on_tick( + &mut server.store, + GENESIS_TIME + config.seconds_per_slot, + &config, + ); + + let (timings, result) = server.process_block(block, ImportTimings::default()).await; + + assert_eq!(new_payload_v4_requests(&mock), 1, "asked exactly once"); + assert!( + timings.stf_speculative_start.is_some(), + "the premise: a speculation was computed while the request was in flight" + ); + assert_eq!( + timings.speculation, + Some(SpeculationOutcome::DiscardedInvalidated) + ); + let err = format!( + "{:?}", + result.expect_err("an INVALID payload does not import") + ); + // The minimal block's own transition error (a fulu block on a phase0 + // state): `on_block` got past every guard before the transition and + // failed in the transition it ran itself, which is where the verdict + // is applied, rather than at a guard in front of it. + assert!( + err.contains("the block's fork matches the state's"), + "the import must fail in the transition, not at a guard before it: {err}" + ); + assert!(!server.store.has_state(&block_root).expect("DB read")); + } + + /// `SYNCING` for a block that is not an optimistic candidate: there is a + /// verdict, but not one the block may be imported on, so it is held and + /// the speculation is dropped. + #[tokio::test] + async fn a_syncing_verdict_for_a_non_candidate_holds_the_block_and_discards_the_speculation() { + use crate::beacon_envelope::tests::{mock_engine, status_json}; + + let (client, mock) = + mock_engine(status_json("SYNCING", None), status_json("SYNCING", None)).await; + // Genesis now, so slot 1 is recent by the wall clock and only an + // execution parent could make the block a candidate. It has none. + let (mut server, block) = server_asking(client, unix_now_ms() / 1000); + let block_root = block.message_hash_tree_root(); + assert!( + !fork_choice::is_optimistic_candidate_block( + &server.store, + server.wall_clock_slot(), + block.slot(), + block.parent_root(), + server.safe_slots_to_import_optimistically, + ), + "the premise: the block is not an optimistic candidate" + ); + + let (timings, result) = server.process_block(block, ImportTimings::default()).await; + + assert_eq!(new_payload_v4_requests(&mock), 1, "asked exactly once"); + assert!( + timings.stf_speculative_start.is_some(), + "the premise: a speculation was computed while the request was in flight" + ); + assert_eq!(timings.speculation, Some(SpeculationOutcome::DiscardedHeld)); + assert_eq!( + result.expect("a hold is not a failure"), + ImportOutcome::Held + ); + assert!(!server.store.has_state(&block_root).expect("DB read")); + } + #[tokio::test] async fn a_tick_leaves_a_block_held_while_a_column_is_still_missing() { let store = beacon_store(GENESIS_TIME, 0); From 75f22b0942c1ef13b01eb81bae3765383acee0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:15:03 -0300 Subject: [PATCH 08/11] fix(fork_choice): speculate only on a block that passes on_block's pre-transition guards speculate_transition ran the transition before any of on_block's guards, so a block on_block refuses for free could make the actor advance the parent's state to the block's slot first. For a req/resp block, which carries no gossip-checked signature, the signature is only checked after that advance. The arrival handler and process_or_pend_block already keep future, finalized and known blocks away from process_block, but nothing kept a block whose parent is off the finalized chain from being speculated. on_block's three checks between the parent lookup and data availability (not from the future, later than the finalized epoch's start, parent on the finalized chain) move into check_timely_and_on_the_finalized_chain, which on_block and speculate_transition now both call, so the two cannot drift. speculate_transition also skips a block that already has a post-state, and reads the LiveChain index without panicking on a storage error. A new test covers each refusal; it fails with the guards removed. The non-candidate SYNCING actor test now sets the store clock to the block's slot, as the arrival handler would, so its block is not refused as one from the future. The speculation counter's docs now say a block refused there is asked about but not counted. --- crates/blockchain/src/import_timing.rs | 5 +- crates/blockchain/src/lib.rs | 12 +- crates/blockchain/src/metrics.rs | 10 +- .../src/beacon/fork_choice.rs | 179 ++++++++++++++---- docs/beacon_engine.md | 9 + docs/metrics.md | 2 +- 6 files changed, 172 insertions(+), 45 deletions(-) diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index 353237ea7..0b5a3e190 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -113,8 +113,9 @@ pub struct VerifyTimings { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SpeculationOutcome { /// Handed to `fork_choice::on_block_speculative` with a verdict that lets it - /// stand in for the transition. A block that then fails one of `on_block`'s - /// earlier checks still counts, since the work was spent either way. + /// stand in for the transition. A speculation is only computed for a block + /// that passes the checks `on_block` makes before its transition, so this + /// is the transition `on_block` took. Used, /// The verdict was `INVALIDATED`, so `on_block` runs its own transition /// with an `invalid()` engine. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index dfb082328..60390cc88 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -5921,8 +5921,18 @@ mod tests { mock_engine(status_json("SYNCING", None), status_json("SYNCING", None)).await; // Genesis now, so slot 1 is recent by the wall clock and only an // execution parent could make the block a candidate. It has none. - let (mut server, block) = server_asking(client, unix_now_ms() / 1000); + let genesis_time = unix_now_ms() / 1000; + let (mut server, block) = server_asking(client, genesis_time); let block_root = block.message_hash_tree_root(); + // The store's clock at the block's slot, as the arrival handler leaves + // it, so the block is not refused as one from the future before the + // verdict is read. The wall clock, which decides candidacy, stays put. + let config = server.store.config(); + fork_choice::on_tick( + &mut server.store, + genesis_time + config.seconds_per_slot, + &config, + ); assert!( !fork_choice::is_optimistic_candidate_block( &server.store, diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index f77279d1c..f8fc66b58 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1510,10 +1510,12 @@ pub fn inc_engine_no_verdict() { /// Speculative state transitions, by what became of them. /// /// With an execution client, the chain actor runs a block's state transition -/// while the `engine_newPayload` round trip is in flight. `outcome` is `used` -/// (handed to `fork_choice::on_block` with a verdict that lets it stand in for -/// the transition; a block that then fails one of `on_block`'s earlier checks -/// still counts, since the work was spent either way), `discarded_invalidated` +/// while the `engine_newPayload` round trip is in flight, for a block that has +/// its parent's post-state and passes the checks `on_block` makes before its +/// own transition (see `fork_choice::speculate_transition`). A block asked +/// about without one is not counted here. `outcome` is `used` (handed to +/// `fork_choice::on_block` with a verdict that lets it stand in for the +/// transition), `discarded_invalidated` /// (the verdict was `INVALIDATED`, so `on_block` runs its own), `discarded_held` /// (no verdict, or a non-candidate `SYNCING`/`ACCEPTED`, so the block is not /// imported) or `discarded_mismatch` (computed for another root or parent; diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 378958ff9..aeb377172 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4905,6 +4905,49 @@ impl SpeculativeTransition { } } +/// The checks [`on_block`] makes between finding the parent's post-state and +/// the data-availability check: the block is not from the future, it is later +/// than the finalized epoch's first slot, and its parent descends from the +/// finalized checkpoint. +/// +/// One function for both callers, so [`speculate_transition`] refuses exactly +/// the blocks `on_block` would refuse here, before either runs a transition. +/// `index` is the `LiveChain` snapshot `on_block` goes on using after these +/// checks, which is why it is a parameter. +fn check_timely_and_on_the_finalized_chain( + store: &Store, + signed_block: &SignedBeaconBlock, + parent_root: Root, + config: &Config, + index: &HashMap, +) -> Result<()> { + // Blocks cannot be in the future. If they are, their consideration must + // be delayed until they are in the past. + verify( + get_current_slot(store, config) >= signed_block.slot(), + "get_current_slot(store) >= block.slot", + )?; + + // Check that block is later than the finalized epoch slot (optimization + // to reduce calls to get_ancestor). + let finalized_checkpoint = store.beacon_finalized_checkpoint(); + let finalized_slot = compute_start_slot_at_epoch(finalized_checkpoint.epoch); + verify( + signed_block.slot() > finalized_slot, + "block.slot > finalized_slot", + )?; + // Check block is a descendant of the finalized block at the checkpoint + // finalized slot. A single-call index: see `get_ancestor`'s documentation + // for why a per-hop lookup would be the wrong trade, which does not apply + // to this one walk. + let finalized_checkpoint_block = + get_checkpoint_block(index, parent_root, finalized_checkpoint.epoch)?; + verify( + finalized_checkpoint.root == finalized_checkpoint_block, + "store.finalized_checkpoint.root == finalized_checkpoint_block", + ) +} + /// Runs `signed_block`'s state transition against its parent's post-state with /// the execution engine assumed valid, without importing anything. /// @@ -4922,10 +4965,23 @@ impl SpeculativeTransition { /// a storage error must not panic it here, and `on_block` meets the same error /// on its own lookup and reports it there. /// -/// None of `on_block`'s own checks run here. They run, unchanged and in the -/// same order, when the speculation is handed to [`on_block_speculative`], so a -/// block that fails one of them is rejected exactly as it is today and the -/// speculation is dropped. +/// Also `None` for a block `on_block` would refuse before its own transition: +/// one that already has a post-state, one from the future, one at or before +/// the finalized epoch's first slot, or one whose parent does not descend from +/// the finalized checkpoint (see [`check_timely_and_on_the_finalized_chain`], +/// which both run). A block arriving over req/resp carries no gossip-checked +/// signature, and the transition checks it only after advancing the parent's +/// state to the block's slot, so without this a block `on_block` rejects for +/// free could make the actor run that advance first. Two of `on_block`'s +/// guards are left to the caller, since neither can fire for a block the actor +/// speculates on: data availability, which `process_block` checks before it +/// asks the execution client, and gloas's parent-payload check, since a gloas +/// block carries no payload to ask about. A storage error on any of these +/// reads is `None` too, for the reason above. +/// +/// `on_block`'s own checks still run, unchanged and in the same order, when the +/// speculation is handed to [`on_block_speculative`]; the ones here only decide +/// whether a transition is worth computing ahead of them. pub fn speculate_transition( store: &Store, signed_block: &SignedBeaconBlock, @@ -4934,7 +4990,13 @@ pub fn speculate_transition( ) -> Option { let block_root = signed_block.message_hash_tree_root(); let parent_root = signed_block.parent_root(); + if !matches!(store.has_state(&block_root), Ok(false)) { + return None; + } let parent_state = store.get_state(&parent_root).ok().flatten()?; + let index = store.get_live_chain().ok()?; + check_timely_and_on_the_finalized_chain(store, signed_block, parent_root, config, &index) + .ok()?; let active_balances = store.active_balance_cache(); let (state, outcome) = transition_block( store, @@ -5042,40 +5104,19 @@ pub fn on_block_speculative( }; let payload_link = rules.payload_link(parent_status); - // Blocks cannot be in the future. If they are, their consideration must - // be delayed until they are in the past. - verify( - get_current_slot(store, config) >= signed_block.slot(), - "get_current_slot(store) >= block.slot", - )?; - - // Check that block is later than the finalized epoch slot (optimization - // to reduce calls to get_ancestor). - let finalized_checkpoint = store.beacon_finalized_checkpoint(); - let finalized_slot = compute_start_slot_at_epoch(finalized_checkpoint.epoch); - verify( - signed_block.slot() > finalized_slot, - "block.slot > finalized_slot", - )?; - // Check block is a descendant of the finalized block at the checkpoint - // finalized slot. A single-call index: see `get_ancestor`'s documentation - // for why a per-hop lookup would be the wrong trade, which does not apply - // to this one walk. - // - // `mut`, and kept alive for the rest of this function: the pre-import - // head computation and the proposer-boost shuffling check further down - // both want this same pre-insertion snapshot of `LiveChain` (`walk_head` - // takes it as a parameter, matching `get_filtered_block_tree`), so it is - // built once here rather than three times over. Once `block_root` itself - // joins the store below, one `insert` keeps this index in step with it - // instead of re-scanning for that alone. + // `mut`, and kept alive for the rest of this function: the finalized-chain + // check just below, the pre-import head computation and the + // proposer-boost shuffling check further down all want this same + // pre-insertion snapshot of `LiveChain` (`walk_head` takes it as a + // parameter, matching `get_filtered_block_tree`), so it is built once here + // rather than three times over. Once `block_root` itself joins the store + // below, one `insert` keeps this index in step with it instead of + // re-scanning for that alone. let mut index = store.block_index(); - let finalized_checkpoint_block = - get_checkpoint_block(&index, parent_root, finalized_checkpoint.epoch)?; - verify( - finalized_checkpoint.root == finalized_checkpoint_block, - "store.finalized_checkpoint.root == finalized_checkpoint_block", - )?; + // Not from the future, later than the finalized epoch's first slot, and on + // the finalized chain. Shared with `speculate_transition`, so a block + // refused here never got a transition there either. + check_timely_and_on_the_finalized_chain(store, &signed_block, parent_root, config, &index)?; // [New in Deneb/Electra] Check if blob data is available. [New in Fulu] // The same check, over column sidecars instead of blobs. Both run before @@ -7685,6 +7726,70 @@ mod tests { ); } + /// A block `on_block` refuses before its own transition never gets one + /// computed ahead of it either. Each case would otherwise transition: the + /// valid child's parent state is there, and the transition itself reads no + /// clock and no finalized checkpoint. + #[test] + fn no_speculation_for_a_block_on_block_refuses_before_its_transition() { + let speculate = |store: &Store, block: &SignedBeaconBlock, config: &Config| { + speculate_transition(store, block, config, &CommitteeCache::default()) + }; + + // The premise: the same block with nothing in the way is speculated. + let (config, store, block) = store_with_valid_child(); + assert!(speculate(&store, &block, &config).is_some()); + + // From the future: the store's clock one slot short of the block's. + let (config, mut store, block) = store_with_valid_child(); + store + .set_time_ms(seconds_to_milliseconds( + config.seconds_per_slot * (block.slot() - 1), + )) + .expect("set time"); + assert!( + speculate(&store, &block, &config).is_none(), + "a block from the future" + ); + + // Off the finalized chain: its parent has a post-state, but is not a + // block the store's `LiveChain` index knows, the shape a retained state + // of a branch finality abandoned leaves. + let (config, mut store, mut block) = store_with_valid_child(); + let stray_parent = Root::repeat_byte(0xab); + let parent_state = store + .get_state(&block.parent_root()) + .expect("get") + .expect("the anchor's post-state"); + store + .insert_state(stray_parent, (*parent_state).clone()) + .expect("insert"); + let SignedBeaconBlock::Phase0(inner) = &mut block else { + unreachable!("phase0"); + }; + inner.message.parent_root = stray_parent; + assert!( + speculate(&store, &block, &config).is_none(), + "a block whose parent is not on the finalized chain" + ); + + // Already imported. + let (config, mut store, block) = store_with_valid_child(); + on_block( + &mut store, + block.clone(), + &config, + &DataAvailability::NotRequired, + &PayloadValidity::NotRequired, + &CommitteeCache::default(), + ) + .expect("the valid child imports"); + assert!( + speculate(&store, &block, &config).is_none(), + "a block that already has a post-state" + ); + } + #[test] fn the_commitments_subtree_index_is_the_bodys_own_position() { // The generalized index the fixture states, minus the offset of a tree diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 47cd6dbe3..9c6a08276 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -106,6 +106,15 @@ and `valid()` for `NOT_REQUIRED`, `VALID` and optimistic alike, and the transition reads nothing else off the store except content-keyed caches. What keeps it harmless: +- A block `on_block` would refuse before its own transition gets no + speculation: one already imported, one from the future, one at or before the + finalized epoch's first slot, or one whose parent is off the finalized chain. + `speculate_transition` runs the same guard function `on_block` does + (`check_timely_and_on_the_finalized_chain`) before transitioning anything. A + block fetched over req/resp carries no gossip-checked signature, and the + transition checks the signature only after advancing the parent's state to + the block's slot, so without this such a block could make the actor run that + advance where `on_block` would have refused it for free. - The result is a local value (`SpeculativeTransition`). The block, its post-state and its fork choice records reach neither the store nor the network until `on_block_speculative` accepts it, and that runs every check `on_block` diff --git a/docs/metrics.md b/docs/metrics.md index fa3614baa..101d28c33 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -269,7 +269,7 @@ Series appear as blocks arrive rather than being seeded, since a third of the ph | Name | Type | Usage | Sample collection event | Labels | Buckets | |------|------|-------|-------------------------|--------|---------| | `lean_block_import_phase_seconds` | Histogram | Time one section of a block's import, or of the arrival carrying it, took | Per section that ran, on each import, hold or failure | phase (see below); source=gossip,sync | 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | -| `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | +| `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client that has its parent's post-state and passes the checks `on_block` makes before its transition (not from the future, past the finalized epoch's start, on the finalized chain); a block refused by one of those is asked about but gets no speculation and is not counted | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | | `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload*`, from the request to the answer, retry ladder included: a pre-gloas block's payload, or from gloas on the payload an envelope reveals. For a block the import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per request sent: each pre-gloas beacon block, and each gloas envelope that passed its consensus checks | outcome=valid,optimistic,invalidated,no_verdict (`no_verdict`: the whole retry ladder with no answer, or one attempt answered with a JSON-RPC error, which is not retried) | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_block_import_cascade_blocks` | Histogram | Blocks one arrival put through the import path (attempts, so a block that ends held or pended counts) | Once per arriving block message | | 1, 2, 3, 5, 8, 16, 32, 64, 128 | From e46a6329913e29db75693f80ed82dbfa700453ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:22:38 -0300 Subject: [PATCH 09/11] refactor(fork_choice): decide a speculation's use from the engine on_block picks Whether a speculation stands in for on_block's transition was decided as "is_for and not Invalidated" in two places, process_block (to count it) and on_block_speculative (to use it), next to an engine selection that said the same thing a third way. A change to which engine a verdict selects would have left the speculation, computed with valid(), standing in under a verdict that no longer selects it. The verdict-to-engine match moves into PayloadValidity::execution_engine, which on_block now calls. SpeculativeTransition keeps the engine it was computed with, and stands_in_for compares it with the verdict's engine. on_block_speculative uses a speculation exactly when stands_in_for says so, and process_block counts "used" with the same call, so the counter cannot disagree with the import. PayloadValidity::is_invalidated, whose two callers this replaces, is gone. --- crates/blockchain/src/lib.rs | 10 +- .../src/beacon/fork_choice.rs | 96 ++++++++++++------- docs/beacon_engine.md | 5 +- 3 files changed, 72 insertions(+), 39 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 60390cc88..8cb621eb3 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1956,13 +1956,15 @@ impl BlockChainServer { // run the transition itself with `invalid()`, exactly as // before. A discarded speculation stays in the `stf` row, so // the transition run twice shows as twice. + // `stands_in_for` is the question `on_block_speculative` + // itself asks, so `used` is counted exactly when it is used. let speculation = speculation.and_then(|speculation| { - let outcome = if !speculation.is_for(block_root, parent_root) { + let outcome = if speculation.stands_in_for(block_root, parent_root, &validity) { + SpeculationOutcome::Used + } else if !speculation.is_for(block_root, parent_root) { SpeculationOutcome::DiscardedMismatch - } else if validity.is_invalidated() { - SpeculationOutcome::DiscardedInvalidated } else { - SpeculationOutcome::Used + SpeculationOutcome::DiscardedInvalidated }; timings.record_speculation(outcome); (outcome == SpeculationOutcome::Used).then_some(speculation) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index aeb377172..22678ea3e 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -665,10 +665,20 @@ pub enum PayloadValidity { } impl PayloadValidity { - /// Whether this is [`PayloadValidity::Invalidated`], the one verdict under - /// which [`on_block`] runs the transition with an `invalid()` engine. - pub fn is_invalidated(&self) -> bool { - matches!(self, Self::Invalidated { .. }) + /// The execution engine [`on_block`]'s state transition consults for a + /// block with this verdict, the one place the verdict reaches the + /// transition. + /// + /// The engine's answer is read the way the specification reads it: an + /// `INVALIDATED` verdict makes `verify_and_notify_new_payload` return false, + /// and everything else makes it return true. It is also what decides + /// whether a [`SpeculativeTransition`], computed with `valid()`, may stand + /// in for that transition, so the two cannot disagree. + pub fn execution_engine(&self) -> stf::ExecutionEngine { + match self { + Self::Invalidated { .. } => stf::ExecutionEngine::invalid(), + Self::NotRequired | Self::Validated | Self::Optimistic => stf::ExecutionEngine::valid(), + } } } @@ -4841,12 +4851,15 @@ pub fn on_block( /// /// # Why it is the transition `on_block` would run /// -/// `on_block` reads the verdict in one place, to pick the -/// [`stf::ExecutionEngine`] the transition consults: `invalid()` for -/// [`PayloadValidity::Invalidated`], `valid()` for every other verdict -/// (`NotRequired`, `Validated` and `Optimistic` alike). So a transition run -/// with `valid()` is byte for byte the one `on_block` runs for every verdict -/// except `Invalidated`, and for that one the speculation is simply not used. +/// `on_block` reads the verdict in one place, +/// [`PayloadValidity::execution_engine`], to pick the [`stf::ExecutionEngine`] +/// the transition consults: `invalid()` for [`PayloadValidity::Invalidated`], +/// `valid()` for every other verdict (`NotRequired`, `Validated` and +/// `Optimistic` alike). So a transition run with `valid()` is byte for byte the +/// one `on_block` runs for every verdict except `Invalidated`, and for that one +/// the speculation is simply not used. [`Self::stands_in_for`] asks that same +/// function rather than naming the verdicts again, so a change to which engine +/// a verdict selects changes which speculations are used along with it. /// /// The inputs are the same too. [`transition_block`] takes the store by shared /// reference and reads two things off it: the epoch-boundary state the @@ -4866,19 +4879,38 @@ pub fn on_block( pub struct SpeculativeTransition { block_root: Root, parent_root: Root, + /// The engine it was computed with, always `valid()`. Kept so + /// [`Self::stands_in_for`] compares it with the one `on_block` would pick + /// rather than restating which verdicts that is. + engine: stf::ExecutionEngine, state: BeaconState, outcome: Result<()>, } impl SpeculativeTransition { /// Whether this was computed for the block with this root and parent root. - /// - /// [`on_block_speculative`] makes the same comparison before using one; a - /// caller asks it first only to tell a mismatch from a use when counting. pub fn is_for(&self, block_root: Root, parent_root: Root) -> bool { self.block_root == block_root && self.parent_root == parent_root } + /// Whether this may stand in for the transition [`on_block`] runs for the + /// block with this root and parent root under `payload_validity`: computed + /// for that block, and with the engine that verdict makes `on_block` + /// consult ([`PayloadValidity::execution_engine`]). + /// + /// [`on_block_speculative`] decides with this, and a caller counting what + /// became of a speculation asks the same question, so the count cannot + /// call a speculation used that `on_block` dropped. + pub fn stands_in_for( + &self, + block_root: Root, + parent_root: Root, + payload_validity: &PayloadValidity, + ) -> bool { + self.is_for(block_root, parent_root) + && self.engine.execution_valid == payload_validity.execution_engine().execution_valid + } + /// Panics unless `fresh`, the transition `on_block` computed itself, agrees /// with this speculation: the same outcome and, when it succeeded, the same /// post-state root. @@ -4998,6 +5030,7 @@ pub fn speculate_transition( check_timely_and_on_the_finalized_chain(store, signed_block, parent_root, config, &index) .ok()?; let active_balances = store.active_balance_cache(); + let engine = stf::ExecutionEngine::valid(); let (state, outcome) = transition_block( store, &parent_state, @@ -5005,13 +5038,14 @@ pub fn speculate_transition( signed_block, true, config, - &stf::ExecutionEngine::valid(), + &engine, committees, &active_balances, ); Some(SpeculativeTransition { block_root, parent_root, + engine, state, outcome, }) @@ -5021,9 +5055,10 @@ pub fn speculate_transition( /// /// Every check `on_block` makes runs unchanged and in the same order. The only /// difference is the one `transition_block` call: when `speculation` was -/// computed for this block's own root and parent root, and the verdict is not -/// `Invalidated` (the one verdict the speculation's `valid()` engine would -/// disagree with), its result stands in for that call. In every other case it +/// computed for this block's own root and parent root, with the engine this +/// verdict makes `on_block` consult ([`SpeculativeTransition::stands_in_for`]; +/// today that is every verdict but `Invalidated`), its result stands in for +/// that call. In every other case it /// is dropped and the transition runs here as it always has, so the /// invalidation path in particular is the one `on_block` has today. /// @@ -5162,20 +5197,14 @@ pub fn on_block_speculative( SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::on_block"), }; - // Check the block is valid and compute the post-state. The engine's answer - // is read the way the specification reads it: an `INVALIDATED` verdict makes - // `verify_and_notify_new_payload` return false, and everything else makes it - // return true. Running the transition even when the verdict is already - // `Invalidated` is deliberate. It costs a merkleization on a path that - // should never run, and it buys the failure arriving from inside - // `process_execution_payload`, which is where the specification puts it and - // where a reviewer checks this code against it. - let engine = match payload_validity { - PayloadValidity::Invalidated { .. } => stf::ExecutionEngine::invalid(), - PayloadValidity::NotRequired | PayloadValidity::Validated | PayloadValidity::Optimistic => { - stf::ExecutionEngine::valid() - } - }; + // Check the block is valid and compute the post-state, with the engine the + // verdict selects (see `PayloadValidity::execution_engine`). Running the + // transition even when the verdict is already `Invalidated` is deliberate. + // It costs a merkleization on a path that should never run, and it buys the + // failure arriving from inside `process_execution_payload`, which is where + // the specification puts it and where a reviewer checks this code against + // it. + let engine = payload_validity.execution_engine(); // // An epoch-crossing block starts from the precomputed boundary state when // one is cached, and from the parent's own post-state otherwise. Both reach @@ -5188,9 +5217,8 @@ pub fn on_block_speculative( // The store's own cache, so every block of this chain shares one entry per // epoch; see `ActiveBalanceCacheExt`. let active_balances = store.active_balance_cache(); - let usable = speculation.filter(|speculation| { - speculation.is_for(block_root, parent_root) && !payload_validity.is_invalidated() - }); + let usable = speculation + .filter(|speculation| speculation.stands_in_for(block_root, parent_root, payload_validity)); let fresh_transition = || { transition_block( store, diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 9c6a08276..4242d1be7 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -131,7 +131,10 @@ keeps it harmless: - It stands in for the transition only when it was computed for this block's root and parent root and the verdict is not `INVALIDATED`. For `INVALIDATED` it is dropped and `on_block` recomputes with `invalid()`, so the invalidation - path is the one described above, unchanged. + path is the one described above, unchanged. The decision is + `SpeculativeTransition::stands_in_for`, which compares the speculation's + engine with `PayloadValidity::execution_engine`, the function `on_block` picks + its own engine with, and the actor counts `used` with that same call. - A block that ends up held (no verdict, or `SYNCING`/`ACCEPTED` for a block that is not an optimistic candidate) drops it. The attempt is counted in `lean_engine_speculation_total{outcome}`. From f38acb2d08ffaa906d50e6e1765b9a771b02f760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:25:10 -0300 Subject: [PATCH 10/11] fix(blockchain): keep building the engine request inside the engine row Two gaps in the import timing rows, which are meant to sum to the end-to-end time: - Building and sending the request (new_payload_request, the payload copy in into_owned, the spawn) ran before the speculative transition started and after engine_start was taken, but engine_start was then moved to the transition's end, so no row claimed it. - When speculate_transition answered None, engine_start still moved to where it returned, so the time it spent declining disappeared too. The engine pair now spans the whole question, from building the request to the later of the answer and the transition's end, and the engine row subtracts the speculative transition that ran inside it, the way stf subtracts writer_wait. The engine row is now what the question cost beyond the transition: the request's construction, plus any wait left. With no speculation it is the whole pair. The overlap tests use the new representation, a new one covers a request with no speculation, and the two actor-level engine tests now assert the engine pair spans the speculation. Also says, in process_block and docs/beacon_engine.md, that the overlap relies on the actor's own multi-thread runtime: the chain actor's Backend::Thread blocks on a multi-thread tokio runtime whose workers run the request, while a current-thread runtime only sends it at the await. --- crates/blockchain/src/import_timing.rs | 81 +++++++++++++++++--------- crates/blockchain/src/lib.rs | 47 ++++++++++++--- docs/beacon_engine.md | 16 +++-- docs/benchmarking.md | 3 +- docs/metrics.md | 4 +- 5 files changed, 109 insertions(+), 42 deletions(-) diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index 0b5a3e190..12631c532 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -208,17 +208,19 @@ pub struct ImportTimings { pub columns_wait_start: Option, pub columns_wait_end: Option, - /// Beacon: how long the import sat waiting for the execution client's - /// verdict with no consensus work of its own left to do. I/O wait, not - /// work. + /// Beacon: what the execution client's verdict cost the import beyond the + /// state transition it overlapped: building and sending the request (a + /// copy of the payload), and then any wait for the answer left once that + /// transition finished. /// - /// The request is sent from a task of its own while the state transition - /// runs (see [`Self::stf_speculative_start`]), so this pair starts where that - /// transition ended and ends when the answer arrived. Both are the same - /// instant, a recorded zero, when there was no request to send (no - /// execution client, or a block with no payload in it: pre-merge or - /// gloas) or when the answer was already in once the transition finished. - /// How long the execution client itself took is + /// This pair spans the whole question, from building the request to the + /// later of the answer and the end of the transition that ran meanwhile + /// (see [`Self::stf_speculative_start`]). That transition lies inside it, + /// and the `engine` row reports the pair minus it, the way `stf` excludes + /// `writer_wait`, so the overlap is counted once, under `stf`. Both ends + /// are the same instant, a recorded zero, when there was no request to + /// send (no execution client, or a block with no payload in it: pre-merge + /// or gloas). How long the execution client itself took is /// `lean_engine_new_payload_seconds`, not this. pub engine_start: Option, pub engine_end: Option, @@ -248,10 +250,12 @@ pub struct ImportTimings { /// work. A discarded one is added too, so the transition an `INVALIDATED` /// verdict makes `on_block` run again shows as the second transition it /// is, and a held block still shows the one it paid for. The `engine` pair - /// starts at this pair's end, so the rows still sum to the end-to-end time - /// instead of counting the overlap twice. `None` whenever no request was - /// sent, which is every import without an execution client and every - /// block with no payload to ask about. + /// spans this one and its row subtracts it, so the rows still sum to the + /// end-to-end time instead of counting the overlap twice. `None` whenever + /// no speculation was computed: every import that sent no request (no + /// execution client, or no payload to ask about), and a request for a + /// block `fork_choice::speculate_transition` declined. Nothing overlapped + /// then, and the `engine` row is its whole pair. pub stf_speculative_start: Option, pub stf_speculative_end: Option, /// Beacon: what became of that speculative transition, set by @@ -439,18 +443,22 @@ impl ImportTimings { /// /// The `stf` row excludes `writer_wait` when one was recorded, since the /// wait is nested inside the `stf` pair; that keeps the rows summing to - /// the end-to-end time and `stf` meaning state-transition work. + /// the end-to-end time and `stf` meaning state-transition work. The + /// speculative transition moves the same way, out of the `engine` pair it + /// ran inside and into `stf`. fn rows(&self) -> Vec { let mut stf = Row::new("stf", self.stf_start, self.stf_end); let writer_wait = Row::new("writer_wait", self.writer_wait_start, self.writer_wait_end); if let (Some(total), Some(wait)) = (stf.elapsed, writer_wait.elapsed) { stf.elapsed = Some(total.saturating_sub(wait)); } + let mut engine = Row::new("engine", self.engine_start, self.engine_end); // Also when the import stopped before running its own transition (a // held block), so the speculation it paid for is not lost. if let Some(spec) = span(self.stf_speculative_start, self.stf_speculative_end) { stf.elapsed = Some(stf.elapsed.unwrap_or_default() + spec); stf.end = stf.end.or(self.stf_speculative_end); + engine.elapsed = engine.elapsed.map(|total| total.saturating_sub(spec)); } vec![ Row::new("decode", self.decode_start, self.decode_end), @@ -471,7 +479,7 @@ impl ImportTimings { self.columns_wait_start, self.columns_wait_end, ), - Row::new("engine", self.engine_start, self.engine_end), + engine, Row::new( "verify_struct", self.verify_structural_start, @@ -976,18 +984,19 @@ mod tests { #[test] fn an_overlapped_transition_is_charged_to_stf_and_not_to_engine() { - // A 100 ms round trip with a 70 ms transition running inside it, then - // 20 ms of import. Un-overlapped, that is 100 + 20 ms; overlapped, the - // engine row keeps the 30 ms that was left and stf gains the 70 ms. + // 5 ms building the request, a 70 ms transition while it is in flight, + // 30 ms more waiting for the answer, then 20 ms of import. The engine + // row keeps the 35 ms the transition did not hide, building the + // request included, and stf gains the 70 ms. let base = Instant::now(); let mut timings = ImportTimings::default(); - (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 0, 70); - (timings.engine_start, timings.engine_end) = pair(base, 70, 30); - (timings.stf_start, timings.stf_end) = pair(base, 100, 20); + (timings.engine_start, timings.engine_end) = pair(base, 0, 105); + (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 5, 70); + (timings.stf_start, timings.stf_end) = pair(base, 105, 20); assert_eq!( row_elapsed(&timings, "engine"), - Some(Duration::from_millis(30)) + Some(Duration::from_millis(35)) ); assert_eq!( row_elapsed(&timings, "stf"), @@ -1000,9 +1009,25 @@ mod tests { .sum(); assert_eq!( rows, - Duration::from_millis(120), - "the rows still tile the span" + Duration::from_millis(125), + "the rows still tile the span, building the request included" + ); + } + + #[test] + fn an_engine_question_with_no_speculation_is_all_engine() { + // A request sent, but nothing computed meanwhile (the block failed a + // guard the speculation shares with `on_block`): every moment of the + // question is the engine's, none of it disappears. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.engine_start, timings.engine_end) = pair(base, 0, 40); + + assert_eq!( + row_elapsed(&timings, "engine"), + Some(Duration::from_millis(40)) ); + assert_eq!(row_elapsed(&timings, "stf"), None); } #[test] @@ -1012,7 +1037,7 @@ mod tests { let base = Instant::now(); let mut timings = ImportTimings::default(); (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 0, 70); - (timings.engine_start, timings.engine_end) = pair(base, 70, 300); + (timings.engine_start, timings.engine_end) = pair(base, 0, 370); assert_eq!( row_elapsed(&timings, "stf"), @@ -1026,8 +1051,8 @@ mod tests { #[test] fn an_engine_with_nothing_to_wait_for_is_a_recorded_zero() { - // No request sent, or the answer was in before the transition ended: - // the row is present and zero, so every beacon block observes it. + // No request sent: the row is present and zero, so every beacon block + // observes it. let base = Instant::now(); let mut timings = ImportTimings::default(); (timings.engine_start, timings.engine_end) = (Some(base), Some(base)); diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 8cb621eb3..77edf2a0e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1826,10 +1826,22 @@ impl BlockChainServer { // Without an engine, or without a request to send (pre-merge // forks, gloas), nothing overlaps and the path is the old one. // - // The `engine` section is what the import waited once it had - // nothing left to do, so it starts as a recorded zero and only - // a request still unanswered when the transition is done - // widens it. The execution client's own time is + // The overlap needs the spawned task to run on another thread + // while this one computes. In the node it does: this actor runs + // on `Backend::Thread`, whose thread blocks on a multi-thread + // tokio runtime of its own, and the task runs on that + // runtime's workers. On a current-thread runtime (a + // `#[tokio::test]` calling this directly) the request only + // leaves at the `await`, and the two add up again: the same + // result, without the gain. + // + // The `engine` pair spans the whole question, from building the + // request to the later of its answer and the transition's end. + // The transition lies inside it and the report subtracts it, so + // the `engine` row is what the question cost beyond the + // transition: building the request, and any wait left once the + // transition is done. A recorded zero without a request. The + // execution client's own time is // `lean_engine_new_payload_seconds`, observed around the // request itself. let no_wait = Instant::now(); @@ -1878,9 +1890,9 @@ impl BlockChainServer { } Err(err) => (Err(NoVerdict::Task(err)), Instant::now()), }; - // The wait begins once the transition is done; an - // answer already in by then leaves it at zero. - timings.engine_start = Some(speculative_end); + // From building the request (`no_wait`) to the later + // of the answer and the transition's end; the report + // takes the transition back out. timings.engine_end = Some(answered_at.max(speculative_end)); answer } @@ -5866,6 +5878,25 @@ mod tests { .count() } + /// The `engine` pair runs from building the request to the later of the + /// answer and the transition's end, so the speculative transition lies + /// inside it and the report can take it back out: nothing the question + /// cost before the transition started (building the request) is lost. + fn assert_engine_pair_spans_the_speculation(timings: &ImportTimings) { + let (Some(engine_start), Some(engine_end)) = (timings.engine_start, timings.engine_end) + else { + panic!("an asked block records the engine pair"); + }; + let (Some(start), Some(end)) = (timings.stf_speculative_start, timings.stf_speculative_end) + else { + panic!("the premise: a speculation was computed"); + }; + assert!( + engine_start <= start && end <= engine_end, + "the engine pair must span the speculative transition" + ); + } + /// `INVALID`: the speculation, computed with a `valid()` engine, is /// dropped, `on_block` runs the transition itself with `invalid()`, and /// nothing is imported. @@ -5893,6 +5924,7 @@ mod tests { timings.stf_speculative_start.is_some(), "the premise: a speculation was computed while the request was in flight" ); + assert_engine_pair_spans_the_speculation(&timings); assert_eq!( timings.speculation, Some(SpeculationOutcome::DiscardedInvalidated) @@ -5953,6 +5985,7 @@ mod tests { timings.stf_speculative_start.is_some(), "the premise: a speculation was computed while the request was in flight" ); + assert_engine_pair_spans_the_speculation(&timings); assert_eq!(timings.speculation, Some(SpeculationOutcome::DiscardedHeld)); assert_eq!( result.expect("a hold is not a failure"), diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 4242d1be7..98c1a0355 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -100,6 +100,13 @@ sends it from a spawned task so the HTTP exchange keeps progressing, and runs transition against its parent's post-state, with the engine assumed valid. The import then costs `max(engine wait, transition)` instead of their sum. +That needs the spawned task to run on another thread while the actor computes, +and in the node it does: the chain actor runs on `Backend::Thread`, whose thread +blocks on a multi-thread tokio runtime of its own, and the task runs on that +runtime's workers. On a current-thread runtime (a `#[tokio::test]` calling +`process_block` directly) the request only leaves at the `await`, so the two add +up again: the same result, without the gain. + Why that is the transition `on_block` would run: `on_block` derives the `ExecutionEngine` from the verdict in one place, `invalid()` for `INVALIDATED` and `valid()` for `NOT_REQUIRED`, `VALID` and optimistic alike, and the @@ -150,10 +157,11 @@ keeps it harmless: The overlap shows in the import timing tree: the transition that ran during the round trip is reported under `stf` (used or discarded), and `engine` reports only -the wait that was left once it finished, a recorded zero when the answer was -already in. Without an execution client, and for forks that have no request to -send (pre-merge, gloas), nothing overlaps, `engine` is zero, and the path is the -old one. How long the execution client itself took to answer is +what the question cost beyond it: building the request (a copy of the +payload) and the wait that was left once the transition finished, the request's +construction alone when the answer was already in. Without an execution client, +and for forks that have no request to send (pre-merge, gloas), nothing overlaps, +`engine` is zero, and the path is the old one. How long the execution client itself took to answer is `lean_engine_new_payload_seconds{outcome}`, timed around every `engine_newPayload*` request this module sends (`timed_new_payload`), a gloas envelope's included. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 0734caf1b..9a7c3fcb3 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -279,7 +279,8 @@ faster than the writer persists them, the wait shows up here rather than in With an execution client the beacon `stf` row also includes a state transition that ran while `engine_newPayload` was in flight, and `engine` then -reports only the wait left after it, zero when the answer was already in. The +reports only what the request cost beyond it: building it, and the wait left +after the transition, which is nothing when the answer was already in. The execution client's own time is `lean_engine_new_payload_seconds`. A replay has no client, so none of this applies to its numbers. diff --git a/docs/metrics.md b/docs/metrics.md index 101d28c33..6dc04bce5 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -260,7 +260,7 @@ The bookkeeping an import triggers (chain-event emission, the finality eviction On the beacon wire, `decode` for `source="gossip"` spans more than its name says. `BlockArrival::decode_start` is still the wire arrival, but `handed_off` is stamped only once the block has a gossip verdict, so the section also covers the cheap, stateless checks, the stateful check's own `spawn_blocking` task, and the verdict's trip back through the p2p actor's mailbox. There is no wait for a free validation slot to attribute here either: `try_acquire_owned` never blocks, and a message arriving with none free is reported `Ignore(Overloaded)` (and still forwarded to the chain actor) rather than queued. A rising `decode` on the beacon wire alone therefore does not mean decoding got slower; check `lean_beacon_gossip_validation_seconds` before assuming so. -`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` round trip the import could not hide behind its own state transition: zero for a block whose answer arrived before that transition finished, the remainder of the round trip for one whose answer did not. How often it is zero depends on the execution client being faster than the transition. The round trip itself is `lean_engine_new_payload_seconds`. The transition that ran meanwhile is charged to `stf` whether or not the import used it, and the log line's `speculation` field says which (`used`, `discarded_invalidated`, `discarded_held` or `discarded_mismatch`), so a block whose `stf` holds two transitions says so. +`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` question the import could not hide behind its own state transition: building the request (a copy of the payload), plus whatever of the round trip was left once that transition finished. For a block whose answer arrived first it is the request's construction alone; how often that is the case depends on the execution client being faster than the transition. The round trip itself is `lean_engine_new_payload_seconds`. The transition that ran meanwhile is charged to `stf` whether or not the import used it, and the log line's `speculation` field says which (`used`, `discarded_invalidated`, `discarded_held` or `discarded_mismatch`), so a block whose `stf` holds two transitions says so. The `source` label has two values, `gossip` and `sync` (req/resp backfill). Two populations are deliberately absent. A block re-delivered to itself because its slot had not started reports under the source it first arrived on, since the hold is already visible as its `defer` section and a third label value would take the block out of the population it belongs to for every section it has left. A block this node built itself is not measured at all: it crossed no wire, so it has no `decode` and an empty `queue` taken when the import began. Both still appear in the log, which names them `deferred` and `local`. @@ -286,7 +286,7 @@ Per-block phases, in the order a block crosses them, plus `total` for a complete | `cascade_wait` | Between the parent's import finishing and this block being popped off the cascade queue | both | | `da_check` | The custody-column availability check itself, not the wait | beacon | | `columns_wait` | Held because custody columns had not all arrived | beacon | -| `engine` | The wait for the `engine_newPayload` verdict left once the state transition that ran meanwhile finished. Observed as `0` for every beacon block whose answer was already in by then, and for every block with no request to send (no execution client, pre-merge, gloas). Non-zero only when the execution client was slower than the transition, or gave no answer (the rest of its retry ladder, block held) | beacon | +| `engine` | What the `engine_newPayload` question cost beyond the state transition that ran meanwhile: building and sending the request (a copy of the payload), and the wait for the verdict left once that transition finished. Observed as `0` for every block with no request to send (no execution client, pre-merge, gloas), and as the request's construction alone for a block whose answer was in before the transition finished. Larger when the execution client was slower than the transition, or gave no answer (the rest of its retry ladder, block held) | beacon | | `verify_struct` | Participant bounds checks and pubkey resolution | lean | | `verify_crypto` | The leanVM multi-message aggregate verification | lean | | `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it. It includes the transition that ran during the `engine_newPayload` round trip, also when that one was then discarded and run again (an `INVALIDATED` verdict) or the block was held | both | From ba404f355824cb57bc949758c0754639c2cc9f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:30:08 -0300 Subject: [PATCH 11/11] test(fork_choice): pin the stray-parent premise and correct two speculation docs - The off-chain case of no_speculation_for_a_block_on_block_refuses_before_its_transition now asserts the stray parent's post-state is readable, so its None can only come from the finalized-chain check. With that check disabled the test fails at this case. - speculate_transition's doc said every storage error on its reads is None. That holds for the known-root, parent-state and LiveChain reads; the finalized checkpoint and the store clock are read as the import path already reads them before process_block speculates. - observe_engine_new_payload's doc still called the engine phase zero when the answer beat the transition; it is now the request's construction. --- crates/blockchain/src/metrics.rs | 5 +++-- .../state_transition/src/beacon/fork_choice.rs | 13 +++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index f8fc66b58..68de7d435 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1543,8 +1543,9 @@ pub fn inc_engine_speculation(outcome: &str) { /// /// For a pre-gloas block the import computes the state transition meanwhile, /// so this is not what the import waited: that is -/// `lean_block_import_phase_seconds`'s `engine` phase, zero whenever the -/// answer was in before the transition finished. This is the execution +/// `lean_block_import_phase_seconds`'s `engine` phase, only the request's +/// construction whenever the answer was in before the transition finished. +/// This is the execution /// client's own processing time. `outcome` is `valid`, `optimistic`, /// `invalidated` or `no_verdict`. A `no_verdict` sample is either the whole /// retry ladder, when no answer came, or a single attempt, when the execution diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 22678ea3e..24342d6df 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -5008,8 +5008,12 @@ fn check_timely_and_on_the_finalized_chain( /// guards are left to the caller, since neither can fire for a block the actor /// speculates on: data availability, which `process_block` checks before it /// asks the execution client, and gloas's parent-payload check, since a gloas -/// block carries no payload to ask about. A storage error on any of these -/// reads is `None` too, for the reason above. +/// block carries no payload to ask about. An error the store reports for the +/// known-root, parent-state or `LiveChain` read is `None` too, for the reason +/// above. The finalized checkpoint and the store's clock are read the way the +/// import path already reads them before it gets here +/// (`process_or_pend_block`, `process_block`), so this adds no panic that path +/// lacks. /// /// `on_block`'s own checks still run, unchanged and in the same order, when the /// speculation is handed to [`on_block_speculative`]; the ones here only decide @@ -7792,6 +7796,11 @@ mod tests { store .insert_state(stray_parent, (*parent_state).clone()) .expect("insert"); + assert!( + store.get_state(&stray_parent).expect("get").is_some(), + "the premise: the stray parent's post-state is readable, so only the \ + finalized-chain check can refuse the block" + ); let SignedBeaconBlock::Phase0(inner) = &mut block else { unreachable!("phase0"); };