Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/blockchain/src/epoch_precompute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ impl BlockChainServer {
let config = self.store.config();
let actor = ctx.actor_ref();
tokio::task::spawn_blocking(move || {
let _running = crate::metrics::EpochPrecomputeRunning::start();
let result = {
let _timing = stf_metrics::time_epoch_precompute();
fork_choice::advance_to_epoch_start(&head_state, key.epoch, &config)
Expand Down
29 changes: 29 additions & 0 deletions crates/blockchain/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,35 @@ impl SlotPosition {

// --- Gauges ---

static LEAN_BEACON_EPOCH_PRECOMPUTE_RUNNING: std::sync::LazyLock<IntGauge> =
std::sync::LazyLock::new(|| {
register_int_gauge!(
"lean_beacon_epoch_precompute_running",
"1 while an epoch precompute worker is running its advance, 0 otherwise"
)
.unwrap()
});

/// Marks an epoch precompute worker as running in
/// `lean_beacon_epoch_precompute_running` until dropped. A guard rather than a
/// set/unset pair so a panicking worker still clears the gauge. At most one
/// worker runs at a time (`epoch_precompute_in_flight`), so a plain 0/1 gauge
/// is exact.
pub struct EpochPrecomputeRunning;

impl EpochPrecomputeRunning {
pub fn start() -> Self {
LEAN_BEACON_EPOCH_PRECOMPUTE_RUNNING.set(1);
Self
}
}

impl Drop for EpochPrecomputeRunning {
fn drop(&mut self) {
LEAN_BEACON_EPOCH_PRECOMPUTE_RUNNING.set(0);
}
}

static LEAN_HEAD_SLOT: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
register_int_gauge!("lean_head_slot", "Latest slot of the lean chain").unwrap()
});
Expand Down
54 changes: 44 additions & 10 deletions crates/net/p2p/src/beacon/column_checks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@
//! [`KZG_BATCH_COLUMNS`], not by how many columns the block has. The
//! prechecks' share is not bounded that way: they take the group's sidecars in
//! turn, one header judgment (a state read, and past the lookahead a state
//! advance) plus a check per sidecar, and how long that holds a permit on a
//! live node is unmeasured. Nothing awaits a permit while it holds one, so
//! groups and sub-batches cannot deadlock the pool between them. The pool is
//! advance) plus a check per sidecar. Both stages' blocking tasks are timed as
//! `kind="column_check"` (`lean_beacon_validation_run_seconds`), so the
//! prechecks' hold is measured only together with the KZG sub-batches', not on
//! its own. Nothing awaits a permit while it holds one, so groups and
//! sub-batches cannot deadlock the pool between them. The pool is
//! shared with the envelope checks (`envelope_checks.rs`, one permit per
//! envelope), and tokio serves waiting requests in arrival order, so an
//! envelope waits behind only the requests queued before it.
Expand All @@ -59,6 +61,7 @@
use std::collections::{HashMap, HashSet};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Instant;

use ethlambda_network_api::P2PToBlockChainRef;
use ethlambda_state_transition::beacon::gossip::column::{
Expand Down Expand Up @@ -191,6 +194,31 @@ const KZG_BATCH_COLUMNS: usize = 16;
/// are checked against.
type Pending = (DataColumnSidecar, PendingKzg);

/// A [`P2PServer::column_check_permits`] permit, counted in
/// `lean_beacon_validation_permits_in_use{pool="column_check"}` for as long as
/// it is held. One value, so the gauge falls wherever the permit is released:
/// at the end of the sub-batch it moved into, or with the group task holding
/// it when nothing was left to verify or a precheck panic cut the group short.
struct HeldPermit {
_permit: OwnedSemaphorePermit,
_in_use: metrics::PoolInUse,
}

/// Wait for a [`P2PServer::column_check_permits`] permit, the wait observed
/// under `lean_beacon_column_check_permit_wait_seconds{kind="column_check"}`.
/// Both a group (its prechecks and first KZG sub-batch) and each further
/// sub-batch take theirs here. `None` only once the semaphore is closed, which
/// it never is.
async fn acquire_permit(permits: &Arc<Semaphore>) -> Option<HeldPermit> {
let waiting_since = Instant::now();
let permit = permits.clone().acquire_owned().await.ok()?;
metrics::observe_column_check_permit_wait("column_check", waiting_since.elapsed());
Some(HeldPermit {
_permit: permit,
_in_use: metrics::PoolInUse::new("column_check"),
})
}

/// What some of one block's sidecars came to: the ones to store, the ones to
/// park, and the reason for each one dropped.
#[derive(Default)]
Expand Down Expand Up @@ -338,12 +366,16 @@ async fn check_group(
blockchain: P2PToBlockChainRef,
permits: Arc<Semaphore>,
group: Vec<DataColumnSidecar>,
permit: OwnedSemaphorePermit,
permit: HeldPermit,
shared: SharedVerdicts,
) {
let group_size = group.len();
let prechecked =
tokio::task::spawn_blocking(move || precheck_group(&store, group, unix_now_ms())).await;
let task = metrics::BlockingTask::new("column_check");
let prechecked = tokio::task::spawn_blocking(move || {
let _run = task.start();
precheck_group(&store, group, unix_now_ms())
})
.await;
let (settled, pending) = match prechecked {
Ok(prechecked) => prechecked,
// Tokio has already caught the panic. A sidecar's own checks catch
Expand All @@ -369,12 +401,12 @@ async fn check_group(
// waits, the first having moved into its batch.
let permit = match own_permit.take() {
Some(permit) => permit,
None => match permits.clone().acquire_owned().await {
Ok(permit) => permit,
None => match acquire_permit(&permits).await {
Some(permit) => permit,
// The semaphore is never closed, so this does not happen
// today. Should it ever, the sidecars not yet sent to a batch
// are lost, and say so rather than vanish.
Err(_) => {
None => {
let lost =
batch.len() + remaining.by_ref().map(|rest| rest.len()).sum::<usize>();
error!(
Expand All @@ -387,8 +419,10 @@ async fn check_group(
},
};
let size = batch.len();
let task = metrics::BlockingTask::new("column_check");
let verified = tokio::task::spawn_blocking(move || {
let _permit = permit;
let _run = task.start();
verify_batch(batch)
});
batches.push((size, verified));
Expand Down Expand Up @@ -486,7 +520,7 @@ pub(crate) fn check_and_forward_then(
for group in groups {
// The semaphore is never closed, so this only returns once a
// permit is free.
let Ok(permit) = permits.clone().acquire_owned().await else {
let Some(permit) = acquire_permit(&permits).await else {
return;
};
tokio::spawn(check_group(
Expand Down
8 changes: 7 additions & 1 deletion crates/net/p2p/src/beacon/envelope_checks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ use ethlambda_storage::Store;
use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope;
use tracing::{debug, error, warn};

use crate::P2PServer;
use crate::{P2PServer, metrics};

/// Whether a fetched envelope goes to the chain actor.
pub(crate) fn forwards(outcome: &Outcome) -> bool {
Expand Down Expand Up @@ -103,15 +103,21 @@ pub(crate) fn check_and_forward_then(
for envelope in envelopes {
// The semaphore is never closed, so this only returns once a
// permit is free.
let waiting_since = std::time::Instant::now();
let Ok(permit) = permits.clone().acquire_owned().await else {
if let Some(done) = done {
done();
}
return;
};
metrics::observe_column_check_permit_wait("envelope_check", waiting_since.elapsed());
let in_use = metrics::PoolInUse::new("column_check");
let task = metrics::BlockingTask::new("envelope_check");
let store = store.clone();
checks.push(tokio::task::spawn_blocking(move || {
let _permit = permit;
let _in_use = in_use;
let _run = task.start();
let outcome = judge(&store, &envelope);
(envelope, outcome)
}));
Expand Down
32 changes: 22 additions & 10 deletions crates/net/p2p/src/beacon/verdict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -601,20 +601,26 @@ fn permits_for<'a>(
server: &'a P2PServer,
object: &Validated,
) -> &'a std::sync::Arc<tokio::sync::Semaphore> {
match pool_of(object) {
"gossip" => &server.gossip_validation_permits,
"attestation" => &server.attestation_validation_permits,
"builder" => &server.builder_validation_permits,
_ => &server.sync_validation_permits,
}
}

/// The label (see [`metrics::VALIDATION_POOLS`]) of the permit pool
/// [`permits_for`] draws `object`'s permit from. The one `match` on the
/// object kind, so the pool metrics and the pool actually used cannot differ.
fn pool_of(object: &Validated) -> &'static str {
match object {
Validated::Block { .. } | Validated::Column(_) | Validated::Envelope(_) => {
&server.gossip_validation_permits
}
Validated::Block { .. } | Validated::Column(_) | Validated::Envelope(_) => "gossip",
Validated::Aggregate { .. }
| Validated::Attestation { .. }
| Validated::PayloadAttestation(_)
| Validated::Operation(_) => &server.attestation_validation_permits,
Validated::ExecutionPayloadBid { .. } | Validated::ProposerPreferences(_) => {
&server.builder_validation_permits
}
Validated::SyncCommitteeMessage { .. } | Validated::SyncContribution(_) => {
&server.sync_validation_permits
}
| Validated::Operation(_) => "attestation",
Validated::ExecutionPayloadBid { .. } | Validated::ProposerPreferences(_) => "builder",
Validated::SyncCommitteeMessage { .. } | Validated::SyncContribution(_) => "sync",
}
}

Expand Down Expand Up @@ -643,7 +649,9 @@ pub(crate) fn spawn_stateful_checks(
id: GossipId,
object: Validated,
) {
let pool = pool_of(&object);
let Ok(permit) = permits_for(server, &object).clone().try_acquire_owned() else {
metrics::inc_validation_permit_exhausted(pool);
let received_at = id.received_at;
let outcome = Outcome::Ignore(IgnoreReason::Overloaded);
report(server, id, outcome);
Expand All @@ -652,8 +660,12 @@ pub(crate) fn spawn_stateful_checks(
};
let store = server.store.clone();
let actor = ctx.actor_ref();
let in_use = metrics::PoolInUse::new(pool);
let task = metrics::BlockingTask::new(id.kind);
tokio::task::spawn_blocking(move || {
let _permit = permit;
let _in_use = in_use;
let _run = task.start();
let mut object = object;
let outcome = guarded(|| object.stateful_checks(&store));
let _ = actor
Expand Down
Loading
Loading