diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index f6aaa1136..cf7b0b8f6 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -26,7 +26,7 @@ libssz.workspace = true spawned-concurrency.workspace = true tokio.workspace = true -tokio-util = { version = "0.7", default-features = false } +tokio-util = { version = "0.7", default-features = false, features = ["rt"] } serde.workspace = true thiserror.workspace = true diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 38f34c39e..cc9699b83 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -19,6 +19,8 @@ //! Gloas is the exception: its payload is revealed in an envelope, so its //! request ([`gloas_new_payload_request`]) needs the block and the envelope. +use std::time::Instant; + use ethlambda_engine::types::PayloadStatusValue; use ethlambda_engine::{EngineClient, EngineError, PayloadStatusV1}; use ethlambda_state_transition::beacon::containers::{self, SignedBeaconBlock}; @@ -178,66 +180,115 @@ pub fn verdict(status: &PayloadStatusV1) -> PayloadValidity { }) } -/// Asks the execution client about `block`'s payload. +/// [`NewPayloadRequest`] with its own copy of the payload, so it can move into +/// a spawned task. /// -/// `Ok(None)` means there was nothing to ask, which is not a failure: -/// [`PayloadValidity::NotRequired`] is the right verdict and the block imports. -/// `Err` means no answer was obtained after the client's whole retry ladder, and -/// `optimistic-sync.md` requires the caller not to import the block and not to -/// touch fork choice. -pub async fn ask( - client: &EngineClient, - block: &SignedBeaconBlock, -) -> Result, EngineError> { - let Some(request) = new_payload_request(block) else { - return Ok(None); - }; - let status = client - .new_payload( - request.execution_payload, - &request.versioned_hashes, - request.parent_beacon_block_root, - &request.execution_requests, - ) - .await?; - Ok(Some(verdict(&status))) +/// The chain actor sends the question from a task of its own and runs the +/// block's state transition meanwhile (see `process_block`), and a task cannot +/// borrow the block the actor goes on using. Only the payload is copied: the +/// rest of the request is already owned, and the block itself, which also +/// carries the attestations and the other operations, is never cloned. +pub struct OwnedNewPayloadRequest { + execution_payload: containers::deneb::ExecutionPayload, + versioned_hashes: Vec, + parent_beacon_block_root: Root, + execution_requests: Vec>, } -/// Asks the execution client about the payload `envelope` reveals for `block`, -/// with `engine_newPayloadV5`. +impl NewPayloadRequest<'_> { + /// Copies the borrowed payload so the request outlives the block. + pub fn into_owned(self) -> OwnedNewPayloadRequest { + OwnedNewPayloadRequest { + execution_payload: self.execution_payload.clone(), + versioned_hashes: self.versioned_hashes, + parent_beacon_block_root: self.parent_beacon_block_root, + execution_requests: self.execution_requests, + } + } +} + +/// Asks the execution client about a block's payload, with a request already +/// built by [`new_payload_request`], which is how the actor sends it from a +/// task while it computes the transition. /// -/// Never `Ok(None)`: unlike [`ask`] there is no "nothing to ask" reading, and a -/// block/envelope pair that does not match is an -/// [`EngineError::EnvelopeMismatch`], which the caller must treat as a refusal -/// to import. -pub async fn ask_envelope( +/// `Err` means no verdict: no answer after the client's whole retry ladder, or +/// a JSON-RPC error answer ([`EngineError::Rpc`]), which is not retried. +/// `optimistic-sync.md` then requires the caller not to import the block and +/// not to touch fork choice. There is no "nothing to ask" reading: a request +/// existing means there was something to ask, and a block with none (pre-merge, +/// gloas) is [`PayloadValidity::NotRequired`] without a call. +pub async fn ask_request( client: &EngineClient, - block: &SignedBeaconBlock, - envelope: &containers::gloas::SignedExecutionPayloadEnvelope, + request: OwnedNewPayloadRequest, ) -> Result { - ask_envelope_status(client, block, envelope) - .await - .map(|status| verdict(&status)) + let call = client.new_payload( + &request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ); + let status = timed_new_payload(call).await?; + Ok(verdict(&status)) } -/// [`ask_envelope`] with the engine's raw answer, for a caller that must tell +/// Awaits `call`, one `engine_newPayload*` request, and observes the time from +/// sending it to its answer as `lean_engine_new_payload_seconds`, under the +/// [`outcome_label`] of the verdict the answer reads as. +/// +/// Every request this module sends goes through here, a block's payload and an +/// envelope's alike, so the metric is the execution client's time for every +/// payload it was asked about. A request refused before it is sent (an +/// envelope that does not match its block) never reaches it. +async fn timed_new_payload( + call: impl Future>, +) -> Result { + let sent = Instant::now(); + let status = call.await; + let validity = status.as_ref().ok().map(verdict); + let outcome = outcome_label(validity.as_ref()); + crate::metrics::observe_engine_new_payload(outcome, sent.elapsed()); + status +} + +/// The `outcome` an answer is counted under: its verdict, or `no_verdict` +/// when there is none. That is either the retry ladder running out without an +/// answer, or a JSON-RPC error answer ([`EngineError::Rpc`]), which ends the +/// call at its first attempt since asking again would get the same refusal. +/// +/// `not_required` is listed for completeness: a status from the execution +/// client always reads as one of the other three verdicts. +pub fn outcome_label(verdict: Option<&PayloadValidity>) -> &'static str { + match verdict { + Some(PayloadValidity::Validated) => "valid", + Some(PayloadValidity::Optimistic) => "optimistic", + Some(PayloadValidity::Invalidated { .. }) => "invalidated", + Some(PayloadValidity::NotRequired) => "not_required", + None => "no_verdict", + } +} + +/// Asks the execution client about the payload `envelope` reveals for `block`, +/// with `engine_newPayloadV5`, and hands back its raw answer. +/// +/// Raw rather than read through [`verdict`], since the caller must tell /// `INVALID_BLOCK_HASH` (the envelope's contents do not hash to its claimed /// block hash, so only that envelope is bad) from `INVALID` (the payload the -/// hash names is bad). +/// hash names is bad). Never "nothing to ask": a block/envelope pair that does +/// not match is an [`EngineError::EnvelopeMismatch`], which the caller must +/// treat as a refusal to import. pub async fn ask_envelope_status( client: &EngineClient, block: &SignedBeaconBlock, envelope: &containers::gloas::SignedExecutionPayloadEnvelope, ) -> Result { let request = gloas_new_payload_request(block, envelope)?; - client - .new_payload_v5( - request.execution_payload, - &request.versioned_hashes, - request.parent_beacon_block_root, - &request.execution_requests, - ) - .await + let call = client.new_payload_v5( + request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ); + timed_new_payload(call).await } #[cfg(test)] diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index a893a651c..4beac299f 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -1190,7 +1190,7 @@ impl BlockChainServer { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::path::PathBuf; use std::sync::Arc; @@ -2189,8 +2189,9 @@ mod tests { /// What the mock execution client answers and has been asked. #[derive(Default)] - struct MockState { - /// The `payloadStatus` object `engine_newPayloadV5` answers with. + pub(crate) struct MockState { + /// The `payloadStatus` object every `engine_newPayload*` answers with: + /// a gloas envelope's V5 and a pre-gloas block's V4 alike. new_payload: String, /// The `payloadStatus` object `engine_forkchoiceUpdated*` answers with. forkchoice: String, @@ -2198,14 +2199,17 @@ mod tests { requests: Vec, } - fn status_json(status: &str, latest_valid_hash: Option) -> String { + pub(crate) fn status_json(status: &str, latest_valid_hash: Option) -> String { let hash = latest_valid_hash.map_or("null".to_string(), |hash| format!("\"{hash:?}\"")); format!(r#"{{"status":"{status}","latestValidHash":{hash},"validationError":null}}"#) } /// A hand-rolled JSON-RPC server answering by method name, like /// `ethlambda-engine`'s wire smoke tests but for many requests. - async fn mock_engine( + /// + /// Shared with the block import tests in `lib.rs`, which ask V4 about a + /// fulu block's payload, so every `engine_newPayload*` gets `new_payload`. + pub(crate) async fn mock_engine( new_payload: String, forkchoice: String, ) -> (EngineClient, Arc>) { @@ -2262,13 +2266,13 @@ mod tests { let result = { let mut state = shared.lock().unwrap(); state.requests.push(body.clone()); - if body.contains("engine_newPayloadV5") { + if body.contains("engine_newPayload") { state.new_payload.clone() } else { state.forkchoice.clone() } }; - let result = if body.contains("engine_newPayloadV5") { + let result = if body.contains("engine_newPayload") { result } else { format!(r#"{{"payloadStatus":{result},"payloadId":null}}"#) @@ -2294,7 +2298,7 @@ mod tests { (client, state) } - fn methods(state: &Arc>) -> Vec { + pub(crate) fn methods(state: &Arc>) -> Vec { state .lock() .unwrap() diff --git a/crates/blockchain/src/import_timing.rs b/crates/blockchain/src/import_timing.rs index 6625a472f..12631c532 100644 --- a/crates/blockchain/src/import_timing.rs +++ b/crates/blockchain/src/import_timing.rs @@ -107,6 +107,38 @@ pub struct VerifyTimings { pub crypto_end: Option, } +/// What became of the state transition a beacon import computed while the +/// execution client's verdict was in flight (see +/// [`ImportTimings::stf_speculative_start`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpeculationOutcome { + /// Handed to `fork_choice::on_block_speculative` with a verdict that lets it + /// stand in for the transition. A speculation is only computed for a block + /// that passes the checks `on_block` makes before its transition, so this + /// is the transition `on_block` took. + Used, + /// The verdict was `INVALIDATED`, so `on_block` runs its own transition + /// with an `invalid()` engine. + DiscardedInvalidated, + /// The block was held without importing: no verdict, or a `SYNCING` or + /// `ACCEPTED` answer for a block that is not an optimistic candidate. + DiscardedHeld, + /// Computed for another root or parent. Not expected from the actor. + DiscardedMismatch, +} + +impl SpeculationOutcome { + /// The `outcome` label of `lean_engine_speculation_total`. + pub fn label(self) -> &'static str { + match self { + Self::Used => "used", + Self::DiscardedInvalidated => "discarded_invalidated", + Self::DiscardedHeld => "discarded_held", + Self::DiscardedMismatch => "discarded_mismatch", + } + } +} + /// Every boundary one block crosses on its way to a post-state. /// /// Carried alongside the block itself: through the actor mailbox in @@ -176,8 +208,20 @@ pub struct ImportTimings { pub columns_wait_start: Option, pub columns_wait_end: Option, - /// Beacon: the `engine_newPayload` round trip, including its retry ladder. - /// I/O wait, not work. + /// Beacon: what the execution client's verdict cost the import beyond the + /// state transition it overlapped: building and sending the request (a + /// copy of the payload), and then any wait for the answer left once that + /// transition finished. + /// + /// This pair spans the whole question, from building the request to the + /// later of the answer and the end of the transition that ran meanwhile + /// (see [`Self::stf_speculative_start`]). That transition lies inside it, + /// and the `engine` row reports the pair minus it, the way `stf` excludes + /// `writer_wait`, so the overlap is counted once, under `stf`. Both ends + /// are the same instant, a recorded zero, when there was no request to + /// send (no execution client, or a block with no payload in it: pre-merge + /// or gloas). How long the execution client itself took is + /// `lean_engine_new_payload_seconds`, not this. pub engine_start: Option, pub engine_end: Option, @@ -197,6 +241,29 @@ pub struct ImportTimings { pub stf_start: Option, pub stf_end: Option, + /// Beacon: a state transition computed while the execution client's + /// verdict was still in flight, whether the import then used it or not. + /// + /// It runs before the `stf` pair above and overlaps the `engine_newPayload` + /// round trip, so it is neither inside `stf_start`..`stf_end` nor a + /// section of its own: the `stf` row adds it, since it is state-transition + /// work. A discarded one is added too, so the transition an `INVALIDATED` + /// verdict makes `on_block` run again shows as the second transition it + /// is, and a held block still shows the one it paid for. The `engine` pair + /// spans this one and its row subtracts it, so the rows still sum to the + /// end-to-end time instead of counting the overlap twice. `None` whenever + /// no speculation was computed: every import that sent no request (no + /// execution client, or no payload to ask about), and a request for a + /// block `fork_choice::speculate_transition` declined. Nothing overlapped + /// then, and the `engine` row is its whole pair. + pub stf_speculative_start: Option, + pub stf_speculative_end: Option, + /// Beacon: what became of that speculative transition, set by + /// [`Self::record_speculation`]. Not a mark, like [`Self::source`]: it + /// travels with them so the import's log line can say whether its `stf` + /// row holds a transition that was discarded and run again. + pub speculation: Option, + /// Beacon: the blocking hand-off of the post-state to the storage crate's /// background writer, which waits while the writer's queue is full. It /// lies inside the `stf` pair; only the report separates the two. @@ -232,6 +299,13 @@ pub struct ImportTimings { } impl ImportTimings { + /// Records what became of this pass's speculative transition, here and in + /// `lean_engine_speculation_total`, so the two cannot disagree. + pub fn record_speculation(&mut self, outcome: SpeculationOutcome) { + self.speculation = Some(outcome); + metrics::inc_engine_speculation(outcome.label()); + } + /// The `source` label this block reports under, or `None` for a block /// that is not reported at all. /// @@ -369,13 +443,23 @@ impl ImportTimings { /// /// The `stf` row excludes `writer_wait` when one was recorded, since the /// wait is nested inside the `stf` pair; that keeps the rows summing to - /// the end-to-end time and `stf` meaning state-transition work. + /// the end-to-end time and `stf` meaning state-transition work. The + /// speculative transition moves the same way, out of the `engine` pair it + /// ran inside and into `stf`. fn rows(&self) -> Vec { let mut stf = Row::new("stf", self.stf_start, self.stf_end); let writer_wait = Row::new("writer_wait", self.writer_wait_start, self.writer_wait_end); if let (Some(total), Some(wait)) = (stf.elapsed, writer_wait.elapsed) { stf.elapsed = Some(total.saturating_sub(wait)); } + let mut engine = Row::new("engine", self.engine_start, self.engine_end); + // Also when the import stopped before running its own transition (a + // held block), so the speculation it paid for is not lost. + if let Some(spec) = span(self.stf_speculative_start, self.stf_speculative_end) { + stf.elapsed = Some(stf.elapsed.unwrap_or_default() + spec); + stf.end = stf.end.or(self.stf_speculative_end); + engine.elapsed = engine.elapsed.map(|total| total.saturating_sub(spec)); + } vec![ Row::new("decode", self.decode_start, self.decode_end), Row::new("queue", self.queue_start, self.queue_end), @@ -395,7 +479,7 @@ impl ImportTimings { self.columns_wait_start, self.columns_wait_end, ), - Row::new("engine", self.engine_start, self.engine_end), + engine, Row::new( "verify_struct", self.verify_structural_start, @@ -570,6 +654,7 @@ impl BlockImportReport { slot_offset_ms = self.slot_offset_ms, bottleneck = self.bottleneck(), da_complete_on_arrival = self.timings.da_complete_on_arrival, + speculation = self.timings.speculation.map(SpeculationOutcome::label), "Block import timing" ); for line in lines { @@ -897,6 +982,84 @@ mod tests { assert_eq!(row_elapsed(&timings, "writer_wait"), None); } + #[test] + fn an_overlapped_transition_is_charged_to_stf_and_not_to_engine() { + // 5 ms building the request, a 70 ms transition while it is in flight, + // 30 ms more waiting for the answer, then 20 ms of import. The engine + // row keeps the 35 ms the transition did not hide, building the + // request included, and stf gains the 70 ms. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.engine_start, timings.engine_end) = pair(base, 0, 105); + (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 5, 70); + (timings.stf_start, timings.stf_end) = pair(base, 105, 20); + + assert_eq!( + row_elapsed(&timings, "engine"), + Some(Duration::from_millis(35)) + ); + assert_eq!( + row_elapsed(&timings, "stf"), + Some(Duration::from_millis(90)) + ); + let rows: Duration = timings + .rows() + .into_iter() + .filter_map(|row| row.elapsed) + .sum(); + assert_eq!( + rows, + Duration::from_millis(125), + "the rows still tile the span, building the request included" + ); + } + + #[test] + fn an_engine_question_with_no_speculation_is_all_engine() { + // A request sent, but nothing computed meanwhile (the block failed a + // guard the speculation shares with `on_block`): every moment of the + // question is the engine's, none of it disappears. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.engine_start, timings.engine_end) = pair(base, 0, 40); + + assert_eq!( + row_elapsed(&timings, "engine"), + Some(Duration::from_millis(40)) + ); + assert_eq!(row_elapsed(&timings, "stf"), None); + } + + #[test] + fn a_held_block_still_charges_its_speculation_to_stf() { + // No verdict: the 70 ms transition ran, then 300 ms of waiting, then + // the block was held without a transition of its own. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.stf_speculative_start, timings.stf_speculative_end) = pair(base, 0, 70); + (timings.engine_start, timings.engine_end) = pair(base, 0, 370); + + assert_eq!( + row_elapsed(&timings, "stf"), + Some(Duration::from_millis(70)) + ); + assert_eq!( + row_elapsed(&timings, "engine"), + Some(Duration::from_millis(300)) + ); + } + + #[test] + fn an_engine_with_nothing_to_wait_for_is_a_recorded_zero() { + // No request sent: the row is present and zero, so every beacon block + // observes it. + let base = Instant::now(); + let mut timings = ImportTimings::default(); + (timings.engine_start, timings.engine_end) = (Some(base), Some(base)); + + assert_eq!(row_elapsed(&timings, "engine"), Some(Duration::ZERO)); + } + #[test] fn a_writer_wait_longer_than_stf_saturates_stf_to_zero() { let base = Instant::now(); diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 251048b7a..77edf2a0e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1,4 +1,6 @@ -use ethlambda_engine::{CustodyColumns, EngineClient, PayloadStatusV1 as EnginePayloadStatus}; +use ethlambda_engine::{ + CustodyColumns, EngineClient, EngineError, PayloadStatusV1 as EnginePayloadStatus, +}; use ethlambda_network_api::{ AggregateArrival, BlockAnnouncement, BlockArrival, BlockChainToP2PRef, BlockSource, DeferredFrom, FetchRequest, InitP2P, @@ -42,13 +44,16 @@ use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Backend, Context, Handler, send_after, }; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, info, trace, warn}; use crate::beacon_columns::{ColumnParking, ParkedColumn}; use crate::beacon_envelope::EnvelopeQueues; use crate::block_builder::ProposerConfig; use crate::events::ChainEvents; -use crate::import_timing::{BlockImportReport, CascadeTimings, HeadTimings, ImportTimings}; +use crate::import_timing::{ + BlockImportReport, CascadeTimings, HeadTimings, ImportTimings, SpeculationOutcome, +}; use crate::store::StoreError; pub use events::{ChainEvent, EventBus, Topic, UnknownTopic}; @@ -765,6 +770,19 @@ pub enum ImportOutcome { Held, } +/// Why a beacon import has no verdict on its block's payload. Either way the +/// block is held, but only the first is the execution client's doing, so the +/// two are logged and counted apart. +enum NoVerdict { + /// No answer after the client's whole retry ladder, or a JSON-RPC error + /// answer. Counted in `lean_engine_no_verdict_total`. + Engine(EngineError), + /// The task that sent the request panicked or was cancelled. A fault in + /// this node, not in the execution client, so it is logged as an error + /// and not counted as the execution client's no-verdict. + Task(tokio::task::JoinError), +} + /// The availability evidence for `block`, or `None` if a column this node /// custodies has not arrived yet. /// @@ -1796,40 +1814,134 @@ impl BlockChainServer { // `is_data_available` runs before `state_transition`, so a // block about to be held for its custody columns is never one // this node asks an execution client about. - timings.engine_start = Some(Instant::now()); - let validity = match &self.engine { - None => fork_choice::PayloadValidity::NotRequired, - Some(client) => match beacon_engine::ask(client, &beacon_block).await { - Ok(None) => fork_choice::PayloadValidity::NotRequired, - Ok(Some(validity)) => validity, - // No answer after the whole ladder. - // `optimistic-sync.md`: a consensus engine MUST NOT - // import the block and MUST NOT apply it to the fork - // choice store. Returning `Held` leaves the block out - // of the store without marking it imported, so the - // cascade does not treat it as having produced a - // post-state. - Err(err) => { - warn!( - %slot, - block_root = %ShortRoot(&block_root.0), - %err, - "No verdict from the execution client; not importing" - ); - metrics::inc_engine_no_verdict(); - timings.engine_end = Some(Instant::now()); - return (timings, Ok(ImportOutcome::Held)); + // + // With an execution client the question is sent from a task + // of its own and this actor computes the block's state + // transition while the answer is awaited, so the two cost + // `max(engine wait, transition)` instead of their sum. The + // result stays a local value: it reaches the store only + // through `fork_choice::on_block_speculative`, after every + // check `on_block` makes and only for a verdict that agrees + // with the transition it assumed (see `SpeculativeTransition`). + // Without an engine, or without a request to send (pre-merge + // forks, gloas), nothing overlaps and the path is the old one. + // + // The overlap needs the spawned task to run on another thread + // while this one computes. In the node it does: this actor runs + // on `Backend::Thread`, whose thread blocks on a multi-thread + // tokio runtime of its own, and the task runs on that + // runtime's workers. On a current-thread runtime (a + // `#[tokio::test]` calling this directly) the request only + // leaves at the `await`, and the two add up again: the same + // result, without the gain. + // + // The `engine` pair spans the whole question, from building the + // request to the later of its answer and the transition's end. + // The transition lies inside it and the report subtracts it, so + // the `engine` row is what the question cost beyond the + // transition: building the request, and any wait left once the + // transition is done. A recorded zero without a request. The + // execution client's own time is + // `lean_engine_new_payload_seconds`, observed around the + // request itself. + let no_wait = Instant::now(); + (timings.engine_start, timings.engine_end) = (Some(no_wait), Some(no_wait)); + // Per pass, like every import-pass mark: a block held here and + // re-imported later must not report the earlier pass's + // speculation as this one's. + (timings.stf_speculative_start, timings.stf_speculative_end) = (None, None); + timings.speculation = None; + let parent_root = beacon_block.parent_root(); + let mut speculation: Option = None; + let request = self.engine.as_ref().and_then(|client| { + let request = beacon_engine::new_payload_request(&beacon_block)?; + Some((client.clone(), request.into_owned())) + }); + let verdict = match request { + None => Ok(fork_choice::PayloadValidity::NotRequired), + Some((client, request)) => { + // Aborted when dropped, so an import abandoned + // mid-await (a timeout or `select!` around this + // handler) does not leave the request running for + // the rest of its retry ladder with nobody to read + // the answer. + let task = AbortOnDropHandle::new(tokio::spawn(async move { + let answer = beacon_engine::ask_request(&client, request).await; + (answer, Instant::now()) + })); + let speculative_start = Instant::now(); + let computed = fork_choice::speculate_transition( + &self.store, + &beacon_block, + &config, + &committees, + ); + let speculative_end = Instant::now(); + // Charged to `stf` whether or not the import ends up + // using it: it is state-transition work either way. + if computed.is_some() { + timings.stf_speculative_start = Some(speculative_start); + timings.stf_speculative_end = Some(speculative_end); } - }, + speculation = computed; + let (answer, answered_at) = match task.await { + Ok((answer, answered_at)) => { + (answer.map_err(NoVerdict::Engine), answered_at) + } + Err(err) => (Err(NoVerdict::Task(err)), Instant::now()), + }; + // From building the request (`no_wait`) to the later + // of the answer and the transition's end; the report + // takes the transition back out. + timings.engine_end = Some(answered_at.max(speculative_end)); + answer + } + }; + // No verdict, whatever the reason. `optimistic-sync.md`: a + // consensus engine MUST NOT import the block and MUST NOT + // apply it to the fork choice store. Returning `Held` leaves + // the block out of the store without marking it imported, so + // the cascade does not treat it as having produced a + // post-state. + let validity = match verdict { + Ok(validity) => validity, + Err(no_verdict) => { + match no_verdict { + // The execution client's doing: no answer after + // the whole ladder, or a JSON-RPC error answer. + NoVerdict::Engine(err) => { + warn!( + %slot, + block_root = %ShortRoot(&block_root.0), + %err, + "No verdict from the execution client; not importing" + ); + metrics::inc_engine_no_verdict(); + } + // This node's: the task that sent the request + // panicked or was cancelled. Not counted as the + // execution client's no-verdict. + NoVerdict::Task(err) => { + error!( + %slot, + block_root = %ShortRoot(&block_root.0), + %err, + "The engine request task failed; not importing" + ); + } + } + if speculation.is_some() { + timings.record_speculation(SpeculationOutcome::DiscardedHeld); + } + return (timings, Ok(ImportOutcome::Held)); + } }; - timings.engine_end = Some(Instant::now()); // An optimistic import is only permitted for a block that // qualifies. A block that does not, and got a NOT_VALIDATED // answer, is not imported at all. if matches!(validity, fork_choice::PayloadValidity::Optimistic) { let current_slot = self.wall_clock_slot(); - let parent_root = beacon_block.parent_root(); if !fork_choice::is_optimistic_candidate_block( &self.store, current_slot, @@ -1844,21 +1956,44 @@ impl BlockChainServer { block and it is not an optimistic candidate" ); metrics::inc_engine_not_optimistic_candidate(); + if speculation.is_some() { + timings.record_speculation(SpeculationOutcome::DiscardedHeld); + } return (timings, Ok(ImportOutcome::Held)); } } + // The speculation is only worth handing on for a verdict its + // `valid()` engine agrees with. `INVALIDATED` makes `on_block` + // run the transition itself with `invalid()`, exactly as + // before. A discarded speculation stays in the `stf` row, so + // the transition run twice shows as twice. + // `stands_in_for` is the question `on_block_speculative` + // itself asks, so `used` is counted exactly when it is used. + let speculation = speculation.and_then(|speculation| { + let outcome = if speculation.stands_in_for(block_root, parent_root, &validity) { + SpeculationOutcome::Used + } else if !speculation.is_for(block_root, parent_root) { + SpeculationOutcome::DiscardedMismatch + } else { + SpeculationOutcome::DiscardedInvalidated + }; + timings.record_speculation(outcome); + (outcome == SpeculationOutcome::Used).then_some(speculation) + }); + // Drop any hand-off span left by an insert outside this call, // so the one taken below is this block's alone. let _ = self.store.take_state_handoff(); timings.stf_start = Some(Instant::now()); - let imported = fork_choice::on_block( + let imported = fork_choice::on_block_speculative( &mut self.store, beacon_block, &config, &evidence, &validity, &committees, + speculation, ); timings.stf_end = Some(Instant::now()); (timings.writer_wait_start, timings.writer_wait_end) = @@ -5645,8 +5780,8 @@ mod tests { /// /// Driven through the real engine client against a closed port, since an /// unreachable execution client is exactly the production shape of this: - /// `beacon_engine::ask` spends its retry ladder and `process_block` - /// answers `Held` with nothing recorded. + /// `beacon_engine::ask_request` spends its retry ladder and + /// `process_block` answers `Held` with nothing recorded. #[tokio::test] async fn a_release_that_gets_no_engine_verdict_puts_the_block_back_on_hold() { let store = beacon_store(GENESIS_TIME, 0); @@ -5703,6 +5838,162 @@ mod tests { ); } + // ----------------------------------------------------------------- + // The state transition overlapped with an answering execution client. + // + // `process_block` sends `engine_newPayloadV4` from a task and computes + // the block's transition meanwhile. These drive it against the mock + // execution client the envelope tests use, for the verdicts that must + // throw that speculation away. The block is the structurally minimal + // fulu one over `bare_state`, so its transition fails whatever the + // verdict; what is under test is the verdict's routing, not the import. + // ----------------------------------------------------------------- + + /// A server asking `client`, holding a post-state for a zero parent root, + /// and a payload-carrying fulu block at slot 1 on that parent. + fn server_asking( + client: EngineClient, + genesis_time: u64, + ) -> (BlockChainServer, SignedBeaconBlock) { + let mut server = beacon_server(beacon_store(genesis_time, 0)); + server.engine = Some(client); + // A block at the zero root too, for `on_block`'s finalized-chain walk. + server + .store + .insert_signed_block(H256::ZERO, bare_block(0, H256::repeat_byte(0xcc))) + .expect("insert"); + server + .store + .insert_state(H256::ZERO, bare_state()) + .expect("insert"); + (server, fulu_block(H256::ZERO, 1, 0)) + } + + fn new_payload_v4_requests( + mock: &Arc>, + ) -> usize { + crate::beacon_envelope::tests::methods(mock) + .iter() + .filter(|method| *method == "engine_newPayloadV4") + .count() + } + + /// The `engine` pair runs from building the request to the later of the + /// answer and the transition's end, so the speculative transition lies + /// inside it and the report can take it back out: nothing the question + /// cost before the transition started (building the request) is lost. + fn assert_engine_pair_spans_the_speculation(timings: &ImportTimings) { + let (Some(engine_start), Some(engine_end)) = (timings.engine_start, timings.engine_end) + else { + panic!("an asked block records the engine pair"); + }; + let (Some(start), Some(end)) = (timings.stf_speculative_start, timings.stf_speculative_end) + else { + panic!("the premise: a speculation was computed"); + }; + assert!( + engine_start <= start && end <= engine_end, + "the engine pair must span the speculative transition" + ); + } + + /// `INVALID`: the speculation, computed with a `valid()` engine, is + /// dropped, `on_block` runs the transition itself with `invalid()`, and + /// nothing is imported. + #[tokio::test] + async fn an_invalid_verdict_discards_the_speculation_and_imports_nothing() { + use crate::beacon_envelope::tests::{mock_engine, status_json}; + + let (client, mock) = + mock_engine(status_json("INVALID", None), status_json("SYNCING", None)).await; + let (mut server, block) = server_asking(client, GENESIS_TIME); + let block_root = block.message_hash_tree_root(); + // The store's clock at the block's slot, so `on_block` gets past its + // future-block guard and reaches the verdict. + let config = server.store.config(); + fork_choice::on_tick( + &mut server.store, + GENESIS_TIME + config.seconds_per_slot, + &config, + ); + + let (timings, result) = server.process_block(block, ImportTimings::default()).await; + + assert_eq!(new_payload_v4_requests(&mock), 1, "asked exactly once"); + assert!( + timings.stf_speculative_start.is_some(), + "the premise: a speculation was computed while the request was in flight" + ); + assert_engine_pair_spans_the_speculation(&timings); + assert_eq!( + timings.speculation, + Some(SpeculationOutcome::DiscardedInvalidated) + ); + let err = format!( + "{:?}", + result.expect_err("an INVALID payload does not import") + ); + // The minimal block's own transition error (a fulu block on a phase0 + // state): `on_block` got past every guard before the transition and + // failed in the transition it ran itself, which is where the verdict + // is applied, rather than at a guard in front of it. + assert!( + err.contains("the block's fork matches the state's"), + "the import must fail in the transition, not at a guard before it: {err}" + ); + assert!(!server.store.has_state(&block_root).expect("DB read")); + } + + /// `SYNCING` for a block that is not an optimistic candidate: there is a + /// verdict, but not one the block may be imported on, so it is held and + /// the speculation is dropped. + #[tokio::test] + async fn a_syncing_verdict_for_a_non_candidate_holds_the_block_and_discards_the_speculation() { + use crate::beacon_envelope::tests::{mock_engine, status_json}; + + let (client, mock) = + mock_engine(status_json("SYNCING", None), status_json("SYNCING", None)).await; + // Genesis now, so slot 1 is recent by the wall clock and only an + // execution parent could make the block a candidate. It has none. + let genesis_time = unix_now_ms() / 1000; + let (mut server, block) = server_asking(client, genesis_time); + let block_root = block.message_hash_tree_root(); + // The store's clock at the block's slot, as the arrival handler leaves + // it, so the block is not refused as one from the future before the + // verdict is read. The wall clock, which decides candidacy, stays put. + let config = server.store.config(); + fork_choice::on_tick( + &mut server.store, + genesis_time + config.seconds_per_slot, + &config, + ); + assert!( + !fork_choice::is_optimistic_candidate_block( + &server.store, + server.wall_clock_slot(), + block.slot(), + block.parent_root(), + server.safe_slots_to_import_optimistically, + ), + "the premise: the block is not an optimistic candidate" + ); + + let (timings, result) = server.process_block(block, ImportTimings::default()).await; + + assert_eq!(new_payload_v4_requests(&mock), 1, "asked exactly once"); + assert!( + timings.stf_speculative_start.is_some(), + "the premise: a speculation was computed while the request was in flight" + ); + assert_engine_pair_spans_the_speculation(&timings); + assert_eq!(timings.speculation, Some(SpeculationOutcome::DiscardedHeld)); + assert_eq!( + result.expect("a hold is not a failure"), + ImportOutcome::Held + ); + assert!(!server.store.has_state(&block_root).expect("DB read")); + } + #[tokio::test] async fn a_tick_leaves_a_block_held_while_a_column_is_still_missing() { let store = beacon_store(GENESIS_TIME, 0); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 34c67398f..68de7d435 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -1507,6 +1507,71 @@ pub fn inc_engine_no_verdict() { LEAN_ENGINE_NO_VERDICT_TOTAL.inc(); } +/// Speculative state transitions, by what became of them. +/// +/// With an execution client, the chain actor runs a block's state transition +/// while the `engine_newPayload` round trip is in flight, for a block that has +/// its parent's post-state and passes the checks `on_block` makes before its +/// own transition (see `fork_choice::speculate_transition`). A block asked +/// about without one is not counted here. `outcome` is `used` (handed to +/// `fork_choice::on_block` with a verdict that lets it stand in for the +/// transition), `discarded_invalidated` +/// (the verdict was `INVALIDATED`, so `on_block` runs its own), `discarded_held` +/// (no verdict, or a non-candidate `SYNCING`/`ACCEPTED`, so the block is not +/// imported) or `discarded_mismatch` (computed for another root or parent; +/// not expected from the actor). A high `discarded_*` share means the overlap +/// is buying nothing. +pub fn inc_engine_speculation(outcome: &str) { + static LEAN_ENGINE_SPECULATION_TOTAL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_counter_vec!( + "lean_engine_speculation_total", + "Speculative state transitions run during the engine round trip, by outcome", + &["outcome"] + ) + .unwrap() + }); + LEAN_ENGINE_SPECULATION_TOTAL + .with_label_values(&[outcome]) + .inc(); +} + +/// How long the execution client took to answer `engine_newPayload*` for a +/// payload, from sending the request to its answer, retry ladder included: +/// a pre-gloas block's payload, and from gloas on the payload an envelope +/// reveals. +/// +/// For a pre-gloas block the import computes the state transition meanwhile, +/// so this is not what the import waited: that is +/// `lean_block_import_phase_seconds`'s `engine` phase, only the request's +/// construction whenever the answer was in before the transition finished. +/// This is the execution +/// client's own processing time. `outcome` is `valid`, `optimistic`, +/// `invalidated` or `no_verdict`. A `no_verdict` sample is either the whole +/// retry ladder, when no answer came, or a single attempt, when the execution +/// client answered with a JSON-RPC error, which is not retried. +pub fn observe_engine_new_payload(outcome: &str, duration: Duration) { + static LEAN_ENGINE_NEW_PAYLOAD_SECONDS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_histogram_vec!( + "lean_engine_new_payload_seconds", + "Time the execution client took to answer engine_newPayload for a block's \ + or an envelope's payload, retries included", + &["outcome"], + // Up to the retry ladder's worst case: three attempts at the + // per-attempt timeout plus the backoff between them. + vec![ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, + 8.0, 12.0, 16.0, 32.0, + ] + ) + .unwrap() + }); + LEAN_ENGINE_NEW_PAYLOAD_SECONDS + .with_label_values(&[outcome]) + .observe(duration.as_secs_f64()); +} + /// Blocks not imported because the execution client has not validated them and /// they do not qualify for an optimistic import. /// diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 238804550..24342d6df 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -664,6 +664,24 @@ pub enum PayloadValidity { }, } +impl PayloadValidity { + /// The execution engine [`on_block`]'s state transition consults for a + /// block with this verdict, the one place the verdict reaches the + /// transition. + /// + /// The engine's answer is read the way the specification reads it: an + /// `INVALIDATED` verdict makes `verify_and_notify_new_payload` return false, + /// and everything else makes it return true. It is also what decides + /// whether a [`SpeculativeTransition`], computed with `valid()`, may stand + /// in for that transition, so the two cannot disagree. + pub fn execution_engine(&self) -> stf::ExecutionEngine { + match self { + Self::Invalidated { .. } => stf::ExecutionEngine::invalid(), + Self::NotRequired | Self::Validated | Self::Optimistic => stf::ExecutionEngine::valid(), + } + } +} + /// Reads an execution client's status as the verdict [`on_block`] takes. /// /// One function for both sources of a status: the real client's JSON-RPC answer @@ -1141,8 +1159,14 @@ fn transition_block( /// is before the first slot of the block's epoch, and the block is at or after /// that slot. The precompute worker stores it under the key /// [`checkpoint_state`] uses for `(block_epoch, parent_root)`, since the two -/// name the same state. Counts a hit or a miss for every such import and for -/// no other. +/// name the same state. +/// +/// Counts a hit or a miss each time a transition of such a block runs, which is +/// not always once per import: a speculation the chain actor computed while the +/// execution client was asked (see [`speculate_transition`]) counts even when it +/// is discarded, so an `INVALIDATED` epoch-crossing block, whose transition +/// runs again under `invalid()`, counts twice, and so does a used speculation in +/// a build with debug assertions, which re-runs the transition to check it. fn precomputed_epoch_state( store: &Store, parent_slot: Slot, @@ -4801,6 +4825,257 @@ pub fn on_block( blob_evidence: &DataAvailability, payload_validity: &PayloadValidity, committees: &CommitteeCache, +) -> Result<()> { + on_block_speculative( + store, + signed_block, + config, + blob_evidence, + payload_validity, + committees, + None, + ) +} + +/// The state transition of one block, computed before the execution client's +/// verdict on its payload is known. +/// +/// Opaque on purpose: the only thing a holder can do with one is hand it to +/// [`on_block_speculative`], which decides whether it may stand in for the +/// transition `on_block` would otherwise run. Until then it is a local value: +/// the block, its post-state and its fork choice records have not reached the +/// store, and dropping it costs only the work. Computing it may still write one +/// content-keyed cache entry, the boundary balances of an epoch-crossing block's +/// checkpoint (see [`speculate_transition`]), which is the entry `on_block`'s +/// own transition would write and is correct whether or not the block imports. +/// +/// # Why it is the transition `on_block` would run +/// +/// `on_block` reads the verdict in one place, +/// [`PayloadValidity::execution_engine`], to pick the [`stf::ExecutionEngine`] +/// the transition consults: `invalid()` for [`PayloadValidity::Invalidated`], +/// `valid()` for every other verdict (`NotRequired`, `Validated` and +/// `Optimistic` alike). So a transition run with `valid()` is byte for byte the +/// one `on_block` runs for every verdict except `Invalidated`, and for that one +/// the speculation is simply not used. [`Self::stands_in_for`] asks that same +/// function rather than naming the verdicts again, so a change to which engine +/// a verdict selects changes which speculations are used along with it. +/// +/// The inputs are the same too. [`transition_block`] takes the store by shared +/// reference and reads two things off it: the epoch-boundary state the +/// precompute worker cached, and, through [`cache_boundary_balances`], the +/// boundary balance cache it fills. Both are keyed by content (a checkpoint, +/// that is an epoch and a block root), so what they hold does not depend on +/// when they are consulted. Everything else is the parent's post-state, the +/// block, the config and the committee and active-balance caches, which are +/// content-keyed as well. The chain actor handles one message at a time, so +/// nothing mutates the store between this computation and `on_block`'s own. +/// +/// That last sentence is an argument, not something the types enforce, so +/// builds with debug assertions (the test suite's `release-fast` included) +/// check it: whenever [`on_block_speculative`] uses a speculation it also runs +/// the transition itself and panics unless both agree on the outcome and, for +/// a success, on the post-state root. +pub struct SpeculativeTransition { + block_root: Root, + parent_root: Root, + /// The engine it was computed with, always `valid()`. Kept so + /// [`Self::stands_in_for`] compares it with the one `on_block` would pick + /// rather than restating which verdicts that is. + engine: stf::ExecutionEngine, + state: BeaconState, + outcome: Result<()>, +} + +impl SpeculativeTransition { + /// Whether this was computed for the block with this root and parent root. + pub fn is_for(&self, block_root: Root, parent_root: Root) -> bool { + self.block_root == block_root && self.parent_root == parent_root + } + + /// Whether this may stand in for the transition [`on_block`] runs for the + /// block with this root and parent root under `payload_validity`: computed + /// for that block, and with the engine that verdict makes `on_block` + /// consult ([`PayloadValidity::execution_engine`]). + /// + /// [`on_block_speculative`] decides with this, and a caller counting what + /// became of a speculation asks the same question, so the count cannot + /// call a speculation used that `on_block` dropped. + pub fn stands_in_for( + &self, + block_root: Root, + parent_root: Root, + payload_validity: &PayloadValidity, + ) -> bool { + self.is_for(block_root, parent_root) + && self.engine.execution_valid == payload_validity.execution_engine().execution_valid + } + + /// Panics unless `fresh`, the transition `on_block` computed itself, agrees + /// with this speculation: the same outcome and, when it succeeded, the same + /// post-state root. + /// + /// Outcomes are compared through their `Debug` rendering, since the error + /// type carries no `PartialEq`, and roots only on success, since a failed + /// transition's partial state is never used. Debug builds only: it doubles + /// the transition, which is the cost the speculation exists to hide. + #[cfg(debug_assertions)] + fn assert_agrees_with(&self, fresh: &(BeaconState, Result<()>)) { + let (fresh_state, fresh_outcome) = fresh; + assert_eq!( + format!("{:?}", self.outcome), + format!("{fresh_outcome:?}"), + "the speculative transition disagrees with on_block's own on the outcome" + ); + if self.outcome.is_ok() { + assert_eq!( + self.state.hash_tree_root(), + fresh_state.hash_tree_root(), + "the speculative transition disagrees with on_block's own on the post-state root" + ); + } + } +} + +/// The checks [`on_block`] makes between finding the parent's post-state and +/// the data-availability check: the block is not from the future, it is later +/// than the finalized epoch's first slot, and its parent descends from the +/// finalized checkpoint. +/// +/// One function for both callers, so [`speculate_transition`] refuses exactly +/// the blocks `on_block` would refuse here, before either runs a transition. +/// `index` is the `LiveChain` snapshot `on_block` goes on using after these +/// checks, which is why it is a parameter. +fn check_timely_and_on_the_finalized_chain( + store: &Store, + signed_block: &SignedBeaconBlock, + parent_root: Root, + config: &Config, + index: &HashMap, +) -> Result<()> { + // Blocks cannot be in the future. If they are, their consideration must + // be delayed until they are in the past. + verify( + get_current_slot(store, config) >= signed_block.slot(), + "get_current_slot(store) >= block.slot", + )?; + + // Check that block is later than the finalized epoch slot (optimization + // to reduce calls to get_ancestor). + let finalized_checkpoint = store.beacon_finalized_checkpoint(); + let finalized_slot = compute_start_slot_at_epoch(finalized_checkpoint.epoch); + verify( + signed_block.slot() > finalized_slot, + "block.slot > finalized_slot", + )?; + // Check block is a descendant of the finalized block at the checkpoint + // finalized slot. A single-call index: see `get_ancestor`'s documentation + // for why a per-hop lookup would be the wrong trade, which does not apply + // to this one walk. + let finalized_checkpoint_block = + get_checkpoint_block(index, parent_root, finalized_checkpoint.epoch)?; + verify( + finalized_checkpoint.root == finalized_checkpoint_block, + "store.finalized_checkpoint.root == finalized_checkpoint_block", + ) +} + +/// Runs `signed_block`'s state transition against its parent's post-state with +/// the execution engine assumed valid, without importing anything. +/// +/// The store is only read, with one exception: an epoch-crossing block's +/// transition caches its boundary checkpoint's balances +/// ([`cache_boundary_balances`]) and counts a precomputed-state lookup, exactly +/// as `on_block`'s own transition would. The entry is keyed by the checkpoint +/// and built from the parent's state alone, so it is right whatever becomes of +/// the block. +/// +/// `None` when the parent has no post-state, the same condition under which +/// `on_block` fails with `block.parent_root in store.block_states`; the caller +/// then gets that failure from `on_block` itself. Also `None` when reading the +/// parent's state fails: this is optional work on the chain actor's thread, so +/// a storage error must not panic it here, and `on_block` meets the same error +/// on its own lookup and reports it there. +/// +/// Also `None` for a block `on_block` would refuse before its own transition: +/// one that already has a post-state, one from the future, one at or before +/// the finalized epoch's first slot, or one whose parent does not descend from +/// the finalized checkpoint (see [`check_timely_and_on_the_finalized_chain`], +/// which both run). A block arriving over req/resp carries no gossip-checked +/// signature, and the transition checks it only after advancing the parent's +/// state to the block's slot, so without this a block `on_block` rejects for +/// free could make the actor run that advance first. Two of `on_block`'s +/// guards are left to the caller, since neither can fire for a block the actor +/// speculates on: data availability, which `process_block` checks before it +/// asks the execution client, and gloas's parent-payload check, since a gloas +/// block carries no payload to ask about. An error the store reports for the +/// known-root, parent-state or `LiveChain` read is `None` too, for the reason +/// above. The finalized checkpoint and the store's clock are read the way the +/// import path already reads them before it gets here +/// (`process_or_pend_block`, `process_block`), so this adds no panic that path +/// lacks. +/// +/// `on_block`'s own checks still run, unchanged and in the same order, when the +/// speculation is handed to [`on_block_speculative`]; the ones here only decide +/// whether a transition is worth computing ahead of them. +pub fn speculate_transition( + store: &Store, + signed_block: &SignedBeaconBlock, + config: &Config, + committees: &CommitteeCache, +) -> Option { + let block_root = signed_block.message_hash_tree_root(); + let parent_root = signed_block.parent_root(); + if !matches!(store.has_state(&block_root), Ok(false)) { + return None; + } + let parent_state = store.get_state(&parent_root).ok().flatten()?; + let index = store.get_live_chain().ok()?; + check_timely_and_on_the_finalized_chain(store, signed_block, parent_root, config, &index) + .ok()?; + let active_balances = store.active_balance_cache(); + let engine = stf::ExecutionEngine::valid(); + let (state, outcome) = transition_block( + store, + &parent_state, + parent_root, + signed_block, + true, + config, + &engine, + committees, + &active_balances, + ); + Some(SpeculativeTransition { + block_root, + parent_root, + engine, + state, + outcome, + }) +} + +/// [`on_block`] with an optional transition computed ahead of time. +/// +/// Every check `on_block` makes runs unchanged and in the same order. The only +/// difference is the one `transition_block` call: when `speculation` was +/// computed for this block's own root and parent root, with the engine this +/// verdict makes `on_block` consult ([`SpeculativeTransition::stands_in_for`]; +/// today that is every verdict but `Invalidated`), its result stands in for +/// that call. In every other case it +/// is dropped and the transition runs here as it always has, so the +/// invalidation path in particular is the one `on_block` has today. +/// +/// With debug assertions on, a speculation that is used is also checked +/// against the transition run here (see [`SpeculativeTransition`]). +pub fn on_block_speculative( + store: &mut Store, + signed_block: SignedBeaconBlock, + config: &Config, + blob_evidence: &DataAvailability, + payload_validity: &PayloadValidity, + committees: &CommitteeCache, + speculation: Option, ) -> Result<()> { let block_root = signed_block.message_hash_tree_root(); let parent_root = signed_block.parent_root(); @@ -4868,40 +5143,19 @@ pub fn on_block( }; let payload_link = rules.payload_link(parent_status); - // Blocks cannot be in the future. If they are, their consideration must - // be delayed until they are in the past. - verify( - get_current_slot(store, config) >= signed_block.slot(), - "get_current_slot(store) >= block.slot", - )?; - - // Check that block is later than the finalized epoch slot (optimization - // to reduce calls to get_ancestor). - let finalized_checkpoint = store.beacon_finalized_checkpoint(); - let finalized_slot = compute_start_slot_at_epoch(finalized_checkpoint.epoch); - verify( - signed_block.slot() > finalized_slot, - "block.slot > finalized_slot", - )?; - // Check block is a descendant of the finalized block at the checkpoint - // finalized slot. A single-call index: see `get_ancestor`'s documentation - // for why a per-hop lookup would be the wrong trade, which does not apply - // to this one walk. - // - // `mut`, and kept alive for the rest of this function: the pre-import - // head computation and the proposer-boost shuffling check further down - // both want this same pre-insertion snapshot of `LiveChain` (`walk_head` - // takes it as a parameter, matching `get_filtered_block_tree`), so it is - // built once here rather than three times over. Once `block_root` itself - // joins the store below, one `insert` keeps this index in step with it - // instead of re-scanning for that alone. + // `mut`, and kept alive for the rest of this function: the finalized-chain + // check just below, the pre-import head computation and the + // proposer-boost shuffling check further down all want this same + // pre-insertion snapshot of `LiveChain` (`walk_head` takes it as a + // parameter, matching `get_filtered_block_tree`), so it is built once here + // rather than three times over. Once `block_root` itself joins the store + // below, one `insert` keeps this index in step with it instead of + // re-scanning for that alone. let mut index = store.block_index(); - let finalized_checkpoint_block = - get_checkpoint_block(&index, parent_root, finalized_checkpoint.epoch)?; - verify( - finalized_checkpoint.root == finalized_checkpoint_block, - "store.finalized_checkpoint.root == finalized_checkpoint_block", - )?; + // Not from the future, later than the finalized epoch's first slot, and on + // the finalized chain. Shared with `speculate_transition`, so a block + // refused here never got a transition there either. + check_timely_and_on_the_finalized_chain(store, &signed_block, parent_root, config, &index)?; // [New in Deneb/Electra] Check if blob data is available. [New in Fulu] // The same check, over column sidecars instead of blobs. Both run before @@ -4947,20 +5201,14 @@ pub fn on_block( SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::on_block"), }; - // Check the block is valid and compute the post-state. The engine's answer - // is read the way the specification reads it: an `INVALIDATED` verdict makes - // `verify_and_notify_new_payload` return false, and everything else makes it - // return true. Running the transition even when the verdict is already - // `Invalidated` is deliberate. It costs a merkleization on a path that - // should never run, and it buys the failure arriving from inside - // `process_execution_payload`, which is where the specification puts it and - // where a reviewer checks this code against it. - let engine = match payload_validity { - PayloadValidity::Invalidated { .. } => stf::ExecutionEngine::invalid(), - PayloadValidity::NotRequired | PayloadValidity::Validated | PayloadValidity::Optimistic => { - stf::ExecutionEngine::valid() - } - }; + // Check the block is valid and compute the post-state, with the engine the + // verdict selects (see `PayloadValidity::execution_engine`). Running the + // transition even when the verdict is already `Invalidated` is deliberate. + // It costs a merkleization on a path that should never run, and it buys the + // failure arriving from inside `process_execution_payload`, which is where + // the specification puts it and where a reviewer checks this code against + // it. + let engine = payload_validity.execution_engine(); // // An epoch-crossing block starts from the precomputed boundary state when // one is cached, and from the parent's own post-state otherwise. Both reach @@ -4973,17 +5221,29 @@ pub fn on_block( // The store's own cache, so every block of this chain shares one entry per // epoch; see `ActiveBalanceCacheExt`. let active_balances = store.active_balance_cache(); - let (mut state, transition) = transition_block( - store, - &parent_state, - parent_root, - &signed_block, - true, - config, - &engine, - committees, - &active_balances, - ); + let usable = speculation + .filter(|speculation| speculation.stands_in_for(block_root, parent_root, payload_validity)); + let fresh_transition = || { + transition_block( + store, + &parent_state, + parent_root, + &signed_block, + true, + config, + &engine, + committees, + &active_balances, + ) + }; + let (mut state, transition) = match usable { + Some(speculation) => { + #[cfg(debug_assertions)] + speculation.assert_agrees_with(&fresh_transition()); + (speculation.state, speculation.outcome) + } + None => fresh_transition(), + }; // `optimistic-sync.md`: a block deemed `INVALIDATED` MUST NOT be included // in the canonical chain. That is stated here, on the verdict, rather than @@ -7135,34 +7395,13 @@ mod tests { assert!(store.is_err()); } - /// [`on_block`]'s early-return guard must read `Store::has_state`, not - /// `Store::has_block`: a block the actor is holding for missing data - /// columns is written with - /// [`Store::insert_pending_block`](ethlambda_storage::Store::insert_pending_block) - /// before it ever imports, which stores the whole signed block in one - /// `BlockHeaders` row (so `has_block` answers true) but writes no - /// `LiveChain` entry (so [`Store::block_index`](ethlambda_storage::Store::block_index), - /// a `LiveChain` scan, does not name it). Guarding on `has_block` instead - /// of `has_state` made `on_block` answer `Ok(())` for a held block - /// without importing it, so every later delivery naming it as an - /// ancestor found it "already known" and gave up without importing - /// anything either -- forever, since nothing ever gave the root a - /// post-state. - /// - /// Delivered timely (the first block for its slot, before the - /// attestation deadline), so this also proves `on_block`'s pre-import - /// `compute_head` call (walking `Store::block_index`, for the new - /// proposer-boost dependent-root gate) tolerates a held block sitting - /// unindexed in the store: that walk never sees it, being a `LiveChain` - /// scan, so it is untouched by the held block's absent post-state. + /// A store anchored one slot before a genuinely valid child block, with the + /// clock at that block's slot, and the block itself. /// - /// Builds a genuinely valid child block (correct proposer, RANDAO reveal, - /// and state root, all real BLS signatures over - /// [`test_state::secret_key_for`]'s key) rather than the zero-signature - /// [`block`] helper, since this exercises the real, signature-checking - /// [`on_block`], not a fixture harness that can skip verification. - #[test] - fn on_block_imports_a_block_already_persisted_with_no_post_state() { + /// Valid means correct proposer, RANDAO reveal and state root, all real BLS + /// signatures over [`test_state::secret_key_for`]'s key, so the real, + /// signature-checking [`on_block`] accepts it. + fn store_with_valid_child() -> (Config, Store, SignedBeaconBlock) { let config = Config::active(); let (anchor_state, anchor_block) = anchor_pair_with(1); let anchor_root = anchor_block.message_hash_tree_root(); @@ -7228,7 +7467,6 @@ mod tests { .to_bytes(), ); let signed_block = draft; - let block_root = signed_block.message_hash_tree_root(); // The store holds the child block already, exactly the shape // `hold_block_for_columns` leaves it in: one `BlockHeaders` row (so @@ -7253,6 +7491,39 @@ mod tests { config.seconds_per_slot * target_slot, )) .expect("set time"); + (config, store, signed_block) + } + + /// [`on_block`]'s early-return guard must read `Store::has_state`, not + /// `Store::has_block`: a block the actor is holding for missing data + /// columns is written with + /// [`Store::insert_pending_block`](ethlambda_storage::Store::insert_pending_block) + /// before it ever imports, which stores the whole signed block in one + /// `BlockHeaders` row (so `has_block` answers true) but writes no + /// `LiveChain` entry (so [`Store::block_index`](ethlambda_storage::Store::block_index), + /// a `LiveChain` scan, does not name it). Guarding on `has_block` instead + /// of `has_state` made `on_block` answer `Ok(())` for a held block + /// without importing it, so every later delivery naming it as an + /// ancestor found it "already known" and gave up without importing + /// anything either -- forever, since nothing ever gave the root a + /// post-state. + /// + /// Delivered timely (the first block for its slot, before the + /// attestation deadline), so this also proves `on_block`'s pre-import + /// `compute_head` call (walking `Store::block_index`, for the new + /// proposer-boost dependent-root gate) tolerates a held block sitting + /// unindexed in the store: that walk never sees it, being a `LiveChain` + /// scan, so it is untouched by the held block's absent post-state. + /// + /// Builds a genuinely valid child block (correct proposer, RANDAO reveal, + /// and state root, all real BLS signatures over + /// [`test_state::secret_key_for`]'s key) rather than the zero-signature + /// [`block`] helper, since this exercises the real, signature-checking + /// [`on_block`], not a fixture harness that can skip verification. + #[test] + fn on_block_imports_a_block_already_persisted_with_no_post_state() { + let (config, mut store, signed_block) = store_with_valid_child(); + let block_root = signed_block.message_hash_tree_root(); store .insert_pending_block(block_root, signed_block.clone()) .expect("insert"); @@ -7284,6 +7555,278 @@ mod tests { ); } + /// Imports the valid child with `validity` and `speculation` and reports + /// everything an import writes that a later reader could tell apart. + fn import_valid_child( + validity: &PayloadValidity, + speculate: impl FnOnce(&Store, &SignedBeaconBlock, &Config) -> Option, + ) -> (Result<()>, Store, Root) { + let (config, mut store, signed_block) = store_with_valid_child(); + let block_root = signed_block.message_hash_tree_root(); + let committees = CommitteeCache::default(); + let speculation = speculate(&store, &signed_block, &config); + let result = on_block_speculative( + &mut store, + signed_block, + &config, + &DataAvailability::NotRequired, + validity, + &committees, + speculation, + ); + (result, store, block_root) + } + + fn honest_speculation( + store: &Store, + block: &SignedBeaconBlock, + config: &Config, + ) -> Option { + speculate_transition(store, block, config, &CommitteeCache::default()) + } + + /// What the store holds after an import, as comparable values. + fn observed( + store: &mut Store, + block_root: Root, + config: &Config, + ) -> ( + Option, + Option<[bool; constants::NUM_BLOCK_TIMELINESS_DEADLINES]>, + Root, + Root, + Vec, + ) { + let post_state_root = store + .get_state(&block_root) + .expect("get") + .map(|state| state.hash_tree_root()); + let mut roots: Vec = store.block_index().keys().copied().collect(); + roots.sort(); + ( + post_state_root, + store.block_timeliness(&block_root), + store.proposer_boost_root(), + get_head(store, config).expect("head"), + roots, + ) + } + + #[test] + fn a_matching_speculation_leaves_the_store_as_importing_without_one() { + let config = Config::active(); + let (plain_result, mut plain, root) = + import_valid_child(&PayloadValidity::NotRequired, |_, _, _| None); + plain_result.expect("the plain import succeeds"); + for validity in [ + PayloadValidity::NotRequired, + PayloadValidity::Validated, + PayloadValidity::Optimistic, + ] { + let (result, mut speculated, speculated_root) = + import_valid_child(&validity, honest_speculation); + result.expect("the speculated import succeeds"); + assert_eq!(speculated_root, root); + assert_eq!( + observed(&mut speculated, root, &config), + observed(&mut plain, root, &config), + "{validity:?}: head, post-state root, timeliness, proposer boost and the \ + block index must be identical" + ); + } + } + + /// A speculation whose outcome is a marker nothing else produces, so the + /// result of the import says whether the speculation or a fresh transition + /// supplied it. + const POISON: &str = "poisoned speculation"; + + fn poisoned( + store: &Store, + block: &SignedBeaconBlock, + config: &Config, + ) -> Option { + let mut speculation = honest_speculation(store, block, config)?; + speculation.outcome = Err(Error::SpecAssert(POISON)); + Some(speculation) + } + + /// Without debug assertions the poisoned outcome is the import's result, + /// which proves the speculation stood in for the transition. With them + /// (every test build, `release-fast` included) `on_block_speculative` also + /// runs the transition and refuses to use a speculation that disagrees, so + /// the same import is the check that the invariant is enforced. + #[test] + #[cfg_attr( + debug_assertions, + should_panic( + expected = "the speculative transition disagrees with on_block's own on the outcome" + ) + )] + fn a_matching_speculation_is_the_transition_on_block_uses() { + let (result, store, root) = import_valid_child(&PayloadValidity::Validated, poisoned); + assert_eq!(failed_assertion(result), Some(POISON)); + assert!( + !store.has_state(&root).unwrap(), + "a failed import writes nothing" + ); + } + + /// The post-state half of the same check: a speculation that succeeded, as + /// the real transition does, but holds a different state. Without debug + /// assertions that state is what the import stores. + #[test] + #[cfg_attr( + debug_assertions, + should_panic( + expected = "the speculative transition disagrees with on_block's own on the post-state root" + ) + )] + fn a_speculation_with_another_post_state_is_caught_in_debug_builds() { + let (_, honest_store, root) = + import_valid_child(&PayloadValidity::Validated, honest_speculation); + let honest_root = honest_store + .get_state(&root) + .expect("get") + .expect("the honest import succeeds") + .hash_tree_root(); + let (result, store, _) = + import_valid_child(&PayloadValidity::Validated, |store, block, config| { + let mut speculation = honest_speculation(store, block, config)?; + speculation + .outcome + .as_ref() + .expect("the transition succeeds"); + speculation.state.latest_block_header_mut().body_root = Root::repeat_byte(0xab); + Some(speculation) + }); + result.expect("the import takes the speculation's outcome"); + let stored = store + .get_state(&root) + .expect("get") + .expect("imported") + .hash_tree_root(); + assert_ne!(stored, honest_root, "the speculation's state was stored"); + } + + #[test] + fn a_speculation_for_another_root_or_parent_is_ignored() { + for tamper in [0u8, 1] { + let (result, store, root) = + import_valid_child(&PayloadValidity::NotRequired, |store, block, config| { + let mut speculation = poisoned(store, block, config)?; + match tamper { + 0 => speculation.block_root = Root::repeat_byte(0xee), + _ => speculation.parent_root = Root::repeat_byte(0xee), + } + Some(speculation) + }); + result.expect("the stale speculation is dropped and the real transition runs"); + assert!(store.has_state(&root).unwrap()); + } + } + + #[test] + fn an_invalidated_verdict_ignores_the_speculation_and_matches_today() { + let invalidated = PayloadValidity::Invalidated { + latest_valid_hash: None, + }; + let (without, plain_store, root) = import_valid_child(&invalidated, |_, _, _| None); + let (with_honest, honest_store, _) = import_valid_child(&invalidated, honest_speculation); + let (with_poison, poison_store, _) = import_valid_child(&invalidated, poisoned); + let without = format!("{:?}", without.expect_err("an invalidated block fails")); + assert_eq!(format!("{:?}", with_honest.expect_err("fails")), without); + assert_eq!( + format!("{:?}", with_poison.expect_err("fails")), + without, + "the speculation must not supply the transition under Invalidated" + ); + for store in [&plain_store, &honest_store, &poison_store] { + assert!(!store.has_state(&root).unwrap()); + } + } + + #[test] + fn no_speculation_without_a_parent_state() { + let (config, store, mut block) = store_with_valid_child(); + let SignedBeaconBlock::Phase0(inner) = &mut block else { + unreachable!("phase0"); + }; + inner.message.parent_root = Root::repeat_byte(0xcd); + assert!( + speculate_transition(&store, &block, &config, &CommitteeCache::default()).is_none() + ); + } + + /// A block `on_block` refuses before its own transition never gets one + /// computed ahead of it either. Each case would otherwise transition: the + /// valid child's parent state is there, and the transition itself reads no + /// clock and no finalized checkpoint. + #[test] + fn no_speculation_for_a_block_on_block_refuses_before_its_transition() { + let speculate = |store: &Store, block: &SignedBeaconBlock, config: &Config| { + speculate_transition(store, block, config, &CommitteeCache::default()) + }; + + // The premise: the same block with nothing in the way is speculated. + let (config, store, block) = store_with_valid_child(); + assert!(speculate(&store, &block, &config).is_some()); + + // From the future: the store's clock one slot short of the block's. + let (config, mut store, block) = store_with_valid_child(); + store + .set_time_ms(seconds_to_milliseconds( + config.seconds_per_slot * (block.slot() - 1), + )) + .expect("set time"); + assert!( + speculate(&store, &block, &config).is_none(), + "a block from the future" + ); + + // Off the finalized chain: its parent has a post-state, but is not a + // block the store's `LiveChain` index knows, the shape a retained state + // of a branch finality abandoned leaves. + let (config, mut store, mut block) = store_with_valid_child(); + let stray_parent = Root::repeat_byte(0xab); + let parent_state = store + .get_state(&block.parent_root()) + .expect("get") + .expect("the anchor's post-state"); + store + .insert_state(stray_parent, (*parent_state).clone()) + .expect("insert"); + assert!( + store.get_state(&stray_parent).expect("get").is_some(), + "the premise: the stray parent's post-state is readable, so only the \ + finalized-chain check can refuse the block" + ); + let SignedBeaconBlock::Phase0(inner) = &mut block else { + unreachable!("phase0"); + }; + inner.message.parent_root = stray_parent; + assert!( + speculate(&store, &block, &config).is_none(), + "a block whose parent is not on the finalized chain" + ); + + // Already imported. + let (config, mut store, block) = store_with_valid_child(); + on_block( + &mut store, + block.clone(), + &config, + &DataAvailability::NotRequired, + &PayloadValidity::NotRequired, + &CommitteeCache::default(), + ) + .expect("the valid child imports"); + assert!( + speculate(&store, &block, &config).is_none(), + "a block that already has a post-state" + ); + } + #[test] fn the_commitments_subtree_index_is_the_bodys_own_position() { // The generalized index the fixture states, minus the offset of a tree diff --git a/crates/blockchain/state_transition/src/metrics.rs b/crates/blockchain/state_transition/src/metrics.rs index 63c029166..8b61ca417 100644 --- a/crates/blockchain/state_transition/src/metrics.rs +++ b/crates/blockchain/state_transition/src/metrics.rs @@ -117,14 +117,18 @@ pub fn time_justified_balances_build() -> TimingGuard { static LEAN_BEACON_EPOCH_PRECOMPUTE_LOOKUPS_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "lean_beacon_epoch_precompute_lookups_total", - "Epoch-crossing block imports, by whether a precomputed epoch-boundary state was cached", + "Epoch-crossing block transitions, by whether a precomputed epoch-boundary state was cached", &["result"] ) .unwrap() }); -/// Count one epoch-crossing import's lookup of a precomputed state: `hit` or -/// `miss`. +/// Count one epoch-crossing transition's lookup of a precomputed state: `hit` +/// or `miss`. +/// +/// Per transition run, not per import: an import that runs a block's +/// transition twice (a discarded speculation, see +/// `fork_choice::precomputed_epoch_state`) looks the state up twice. pub fn inc_epoch_precompute_lookups(result: &str) { LEAN_BEACON_EPOCH_PRECOMPUTE_LOOKUPS_TOTAL .with_label_values(&[result]) diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index c0cec5c4f..98c1a0355 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -91,6 +91,80 @@ already `INVALIDATED`, which costs one merkleization on a path that should never run and buys the failure arriving where a reviewer checks this code against the specification. +### The transition runs while the answer is in flight + +With an execution client, `process_block` does not wait idle for the answer. It +builds the request on the actor (copying only the payload, never the block), +sends it from a spawned task so the HTTP exchange keeps progressing, and runs +`fork_choice::speculate_transition` on the actor meanwhile: the block's state +transition against its parent's post-state, with the engine assumed valid. The +import then costs `max(engine wait, transition)` instead of their sum. + +That needs the spawned task to run on another thread while the actor computes, +and in the node it does: the chain actor runs on `Backend::Thread`, whose thread +blocks on a multi-thread tokio runtime of its own, and the task runs on that +runtime's workers. On a current-thread runtime (a `#[tokio::test]` calling +`process_block` directly) the request only leaves at the `await`, so the two add +up again: the same result, without the gain. + +Why that is the transition `on_block` would run: `on_block` derives the +`ExecutionEngine` from the verdict in one place, `invalid()` for `INVALIDATED` +and `valid()` for `NOT_REQUIRED`, `VALID` and optimistic alike, and the +transition reads nothing else off the store except content-keyed caches. What +keeps it harmless: + +- A block `on_block` would refuse before its own transition gets no + speculation: one already imported, one from the future, one at or before the + finalized epoch's first slot, or one whose parent is off the finalized chain. + `speculate_transition` runs the same guard function `on_block` does + (`check_timely_and_on_the_finalized_chain`) before transitioning anything. A + block fetched over req/resp carries no gossip-checked signature, and the + transition checks the signature only after advancing the parent's state to + the block's slot, so without this such a block could make the actor run that + advance where `on_block` would have refused it for free. +- The result is a local value (`SpeculativeTransition`). The block, its + post-state and its fork choice records reach neither the store nor the network + until `on_block_speculative` accepts it, and that runs every check `on_block` + makes, unchanged and in the same order. The one write computing it can make is + an epoch-crossing block's boundary balances, a cache entry keyed by the + checkpoint and built from the parent's state alone, so it is the entry + `on_block`'s own transition would write and is right whatever becomes of the + block. +- The argument above rests on the actor handling one message at a time, which + no type enforces. Builds with debug assertions (the test suite's + `release-fast` among them) therefore run the transition again whenever a + speculation is used and panic unless both agree on the outcome and the + post-state root. +- It stands in for the transition only when it was computed for this block's + root and parent root and the verdict is not `INVALIDATED`. For `INVALIDATED` + it is dropped and `on_block` recomputes with `invalid()`, so the invalidation + path is the one described above, unchanged. The decision is + `SpeculativeTransition::stands_in_for`, which compares the speculation's + engine with `PayloadValidity::execution_engine`, the function `on_block` picks + its own engine with, and the actor counts `used` with that same call. +- A block that ends up held (no verdict, or `SYNCING`/`ACCEPTED` for a block + that is not an optimistic candidate) drops it. The attempt is counted in + `lean_engine_speculation_total{outcome}`. +- A panicked or cancelled request task holds the block too, since there is no + verdict, but it is a fault in this node rather than in the execution client: + it is logged as an error and is not counted in `lean_engine_no_verdict_total`. +- The request task is aborted if the import is dropped before the answer is + read, so an abandoned import never leaves a request running out its retry + ladder. +- Reading the parent's state for the speculation never panics the actor: a + storage error there skips the speculation, and `on_block` meets the same error + on its own lookup. + +The overlap shows in the import timing tree: the transition that ran during the +round trip is reported under `stf` (used or discarded), and `engine` reports only +what the question cost beyond it: building the request (a copy of the +payload) and the wait that was left once the transition finished, the request's +construction alone when the answer was already in. Without an execution client, +and for forks that have no request to send (pre-merge, gloas), nothing overlaps, +`engine` is zero, and the path is the old one. How long the execution client itself took to answer is +`lean_engine_new_payload_seconds{outcome}`, timed around every `engine_newPayload*` +request this module sends (`timed_new_payload`), a gloas envelope's included. + ## The verdicts `optimistic-sync.md` groups the five wire statuses into three outcomes: diff --git a/docs/benchmarking.md b/docs/benchmarking.md index ae50db389..9a7c3fcb3 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -277,13 +277,22 @@ is state-transition work and the two phases do not overlap. When blocks import faster than the writer persists them, the wait shows up here rather than in `stf`. +With an execution client the beacon `stf` row also includes a state +transition that ran while `engine_newPayload` was in flight, and `engine` then +reports only what the request cost beyond it: building it, and the wait left +after the transition, which is nothing when the answer was already in. The +execution client's own time is `lean_engine_new_payload_seconds`. A replay has +no client, so none of this applies to its numbers. + ### What is excluded, and why Replay runs nothing that a corpus already answers for: -- **No execution client**, so `engine` reports a near-zero section with nothing - to call and `fcu` never runs. On a live follower paired with one, `engine` was - 31ms p50, about 0.5% of an import; a replay's numbers are complete without it. +- **No execution client**, so `engine` reports a zero section with nothing to + call and `fcu` never runs. On a live follower paired with one, the + `engine_newPayload` round trip was 31ms p50, about 0.5% of an import, before + the state transition was overlapped with it; a replay's numbers are complete + without it. - **An empty custody set**, so `columns_wait` never runs and no block is ever held waiting on data availability. A corpus supplies every block directly; there are no columns to wait for. diff --git a/docs/metrics.md b/docs/metrics.md index c9264b677..6dc04bce5 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -260,7 +260,7 @@ The bookkeeping an import triggers (chain-event emission, the finality eviction On the beacon wire, `decode` for `source="gossip"` spans more than its name says. `BlockArrival::decode_start` is still the wire arrival, but `handed_off` is stamped only once the block has a gossip verdict, so the section also covers the cheap, stateless checks, the stateful check's own `spawn_blocking` task, and the verdict's trip back through the p2p actor's mailbox. There is no wait for a free validation slot to attribute here either: `try_acquire_owned` never blocks, and a message arriving with none free is reported `Ignore(Overloaded)` (and still forwarded to the chain actor) rather than queued. A rising `decode` on the beacon wire alone therefore does not mean decoding got slower; check `lean_beacon_gossip_validation_seconds` before assuming so. -`engine` and `fcu` are execution-client round trips. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. +`engine` and `fcu` are execution-client waits. They are I/O waits rather than work, so a node whose import time is dominated by them is waiting on its execution client, not spending CPU. `engine` is only the part of the `engine_newPayload` question the import could not hide behind its own state transition: building the request (a copy of the payload), plus whatever of the round trip was left once that transition finished. For a block whose answer arrived first it is the request's construction alone; how often that is the case depends on the execution client being faster than the transition. The round trip itself is `lean_engine_new_payload_seconds`. The transition that ran meanwhile is charged to `stf` whether or not the import used it, and the log line's `speculation` field says which (`used`, `discarded_invalidated`, `discarded_held` or `discarded_mismatch`), so a block whose `stf` holds two transitions says so. The `source` label has two values, `gossip` and `sync` (req/resp backfill). Two populations are deliberately absent. A block re-delivered to itself because its slot had not started reports under the source it first arrived on, since the hold is already visible as its `defer` section and a third label value would take the block out of the population it belongs to for every section it has left. A block this node built itself is not measured at all: it crossed no wire, so it has no `decode` and an empty `queue` taken when the import began. Both still appear in the log, which names them `deferred` and `local`. @@ -269,6 +269,8 @@ Series appear as blocks arrive rather than being seeded, since a third of the ph | Name | Type | Usage | Sample collection event | Labels | Buckets | |------|------|-------|-------------------------|--------|---------| | `lean_block_import_phase_seconds` | Histogram | Time one section of a block's import, or of the arrival carrying it, took | Per section that ran, on each import, hold or failure | phase (see below); source=gossip,sync | 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | +| `lean_engine_speculation_total` | Counter | State transitions the chain actor ran while the `engine_newPayload` round trip was in flight, by what became of them | Per beacon block asked of an execution client that has its parent's post-state and passes the checks `on_block` makes before its transition (not from the future, past the finalized epoch's start, on the finalized chain); a block refused by one of those is asked about but gets no speculation and is not counted | outcome=used,discarded_invalidated,discarded_held,discarded_mismatch | | +| `lean_engine_new_payload_seconds` | Histogram | Time the execution client took to answer `engine_newPayload*`, from the request to the answer, retry ladder included: a pre-gloas block's payload, or from gloas on the payload an envelope reveals. For a block the import runs the state transition meanwhile, so this is the execution client's processing time, not what the import waited (the `engine` phase) | Per request sent: each pre-gloas beacon block, and each gloas envelope that passed its consensus checks | outcome=valid,optimistic,invalidated,no_verdict (`no_verdict`: the whole retry ladder with no answer, or one attempt answered with a JSON-RPC error, which is not retried) | 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 6, 8, 12, 16, 32 | | `lean_block_import_cascade_blocks` | Histogram | Blocks one arrival put through the import path (attempts, so a block that ends held or pended counts) | Once per arriving block message | | 1, 2, 3, 5, 8, 16, 32, 64, 128 | Per-block phases, in the order a block crosses them, plus `total` for a completed import: @@ -284,10 +286,10 @@ Per-block phases, in the order a block crosses them, plus `total` for a complete | `cascade_wait` | Between the parent's import finishing and this block being popped off the cascade queue | both | | `da_check` | The custody-column availability check itself, not the wait | beacon | | `columns_wait` | Held because custody columns had not all arrived | beacon | -| `engine` | The `engine_newPayload` round trip, including its retry ladder | beacon | +| `engine` | What the `engine_newPayload` question cost beyond the state transition that ran meanwhile: building and sending the request (a copy of the payload), and the wait for the verdict left once that transition finished. Observed as `0` for every block with no request to send (no execution client, pre-merge, gloas), and as the request's construction alone for a block whose answer was in before the transition finished. Larger when the execution client was slower than the transition, or gave no answer (the rest of its retry ladder, block held) | beacon | | `verify_struct` | Participant bounds checks and pubkey resolution | lean | | `verify_crypto` | The leanVM multi-message aggregate verification | lean | -| `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it | both | +| `stf` | The state transition. On beacon this bundles the transition and the state root, and excludes `writer_wait`, which is nested inside it. It includes the transition that ran during the `engine_newPayload` round trip, also when that one was then discarded and run again (an `INVALIDATED` verdict) or the block was held | both | | `writer_wait` | Beacon only: the importer's blocking hand-off of the post-state to the storage crate's background writer, which waits while the writer's queue is full. Subtracted from `stf`, so `stf` and `writer_wait` do not overlap | beacon | | `db_write` | The block write and handing the post-state off to the storage crate's background writer; the state's own encode/diff/commit cost is `lean_state_write_seconds` instead, not this row | lean | | `fc_head` | `update_head` | lean | @@ -555,7 +557,7 @@ leanMetrics spec. | Name | Type | Usage | Sample collection event | Labels | Buckets | |------|------|-------|-------------------------|--------|---------| -| `lean_beacon_epoch_precompute_lookups_total` | Counter | Epoch-crossing imports, by whether a precomputed boundary state was cached | On each import whose parent is before the block's epoch start and whose block is at or after it | result=hit,miss | | +| `lean_beacon_epoch_precompute_lookups_total` | Counter | Epoch-crossing block transitions, by whether a precomputed boundary state was cached | On each state transition of a block whose parent is before the block's epoch start and whose block is at or after it. Usually once per import; twice when the import runs the transition again (a speculation computed during the `engine_newPayload` round trip and then discarded for an `INVALIDATED` verdict, or re-checked in a debug build), and once for a block held after its speculation ran | result=hit,miss | | | `lean_beacon_epoch_precompute_seconds` | Histogram | Worker time to advance, flush and hash one state | When a precompute worker finishes | | 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 8 | | `lean_beacon_epoch_precompute_started_total` | Counter | Workers started, by trigger | When a worker is spawned | trigger=head,timer | |