From 1b974dc0b7fc02ea352dfefffc98556da3b1df4c Mon Sep 17 00:00:00 2001 From: Ayelet Zilber Date: Wed, 20 Aug 2025 13:59:00 +0300 Subject: [PATCH] apollo_storage: add panels --- Cargo.lock | 1 + crates/apollo_dashboard/Cargo.toml | 6 +---- .../resources/dev_grafana.json | 25 +++++++++++++++++-- crates/apollo_dashboard/src/panels/storage.rs | 16 ++++++++++-- crates/apollo_storage/Cargo.toml | 1 + crates/apollo_storage/src/lib.rs | 7 ++++-- crates/apollo_storage/src/metrics.rs | 14 +++++++++++ crates/apollo_storage/src/state/mod.rs | 5 ++-- 8 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 crates/apollo_storage/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 958c3863bae..e5191942409 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2117,6 +2117,7 @@ name = "apollo_storage" version = "0.15.0-rc.2" dependencies = [ "apollo_config", + "apollo_metrics", "apollo_proc_macros", "apollo_test_utils", "assert_matches", diff --git a/crates/apollo_dashboard/Cargo.toml b/crates/apollo_dashboard/Cargo.toml index c59dc79c1dd..c2d1d4b47e6 100644 --- a/crates/apollo_dashboard/Cargo.toml +++ b/crates/apollo_dashboard/Cargo.toml @@ -52,8 +52,4 @@ apollo_l1_gas_price = { workspace = true, features = ["testing"] } apollo_l1_provider = { workspace = true, features = ["testing"] } apollo_mempool = { workspace = true, features = ["testing"] } apollo_mempool_p2p = { workspace = true, features = ["testing"] } -apollo_state_sync_metrics = { workspace = true, features = ["testing"] } - -[package.metadata.cargo-machete] -# `apollo_storage` will be used when metrics are addded. -ignored = ["apollo_storage"] +apollo_state_sync_metrics = { workspace = true, features = ["testing"] } \ No newline at end of file diff --git a/crates/apollo_dashboard/resources/dev_grafana.json b/crates/apollo_dashboard/resources/dev_grafana.json index f40a1572c06..c9bd47e8bae 100644 --- a/crates/apollo_dashboard/resources/dev_grafana.json +++ b/crates/apollo_dashboard/resources/dev_grafana.json @@ -941,7 +941,28 @@ "extra_params": {} } ], - "Storage": [], + "Storage": [ + { + "title": "storage_append_thin_state_diff_latency_seconds", + "description": "Latency to append thin state diff in storage (secs)", + "type": "timeseries", + "exprs": [ + "histogram_quantile(0.50, sum by (le) (rate(storage_append_thin_state_diff_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))", + "histogram_quantile(0.95, sum by (le) (rate(storage_append_thin_state_diff_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))" + ], + "extra_params": {} + }, + { + "title": "storage_commit_latency_seconds", + "description": "Latency to commit changes in storage (secs)", + "type": "timeseries", + "exprs": [ + "histogram_quantile(0.50, sum by (le) (rate(storage_commit_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))", + "histogram_quantile(0.95, sum by (le) (rate(storage_commit_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))" + ], + "extra_params": {} + } + ], "MempoolP2p": [ { "title": "apollo_mempool_p2p_num_connected_peers", @@ -2537,4 +2558,4 @@ } ] } -} +} \ No newline at end of file diff --git a/crates/apollo_dashboard/src/panels/storage.rs b/crates/apollo_dashboard/src/panels/storage.rs index d50dbf3a05f..aba20cf5b8c 100644 --- a/crates/apollo_dashboard/src/panels/storage.rs +++ b/crates/apollo_dashboard/src/panels/storage.rs @@ -1,5 +1,17 @@ -use crate::dashboard::Row; +use apollo_storage::metrics::{STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, STORAGE_COMMIT_LATENCY}; + +use crate::dashboard::{Panel, PanelType, Row}; + +fn get_storage_append_thin_state_diff_latency() -> Panel { + Panel::from_hist(STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, PanelType::TimeSeries) +} +fn get_storage_commit_latency() -> Panel { + Panel::from_hist(STORAGE_COMMIT_LATENCY, PanelType::TimeSeries) +} pub(crate) fn get_storage_row() -> Row { - Row::new("Storage", vec![]) + Row::new( + "Storage", + vec![get_storage_append_thin_state_diff_latency(), get_storage_commit_latency()], + ) } diff --git a/crates/apollo_storage/Cargo.toml b/crates/apollo_storage/Cargo.toml index 2a4d8ac1a5a..e677a8a0f7e 100644 --- a/crates/apollo_storage/Cargo.toml +++ b/crates/apollo_storage/Cargo.toml @@ -17,6 +17,7 @@ required-features = ["clap", "statistical"] [dependencies] apollo_config.workspace = true +apollo_metrics.workspace = true apollo_proc_macros.workspace = true byteorder.workspace = true cairo-lang-casm = { workspace = true, features = ["parity-scale-codec"] } diff --git a/crates/apollo_storage/src/lib.rs b/crates/apollo_storage/src/lib.rs index fb6fb179c18..7bcba712d79 100644 --- a/crates/apollo_storage/src/lib.rs +++ b/crates/apollo_storage/src/lib.rs @@ -83,6 +83,8 @@ pub mod class_manager; pub mod compiled_class; #[cfg(feature = "document_calls")] pub mod document_calls; +#[allow(missing_docs)] +pub mod metrics; pub mod storage_metrics; // TODO(yair): Make the compression_utils module pub(crate) or extract it from the crate. #[doc(hidden)] @@ -109,7 +111,7 @@ use std::sync::Arc; use apollo_config::dumping::{prepend_sub_config_name, ser_param, SerializeConfig}; use apollo_config::{ParamPath, ParamPrivacyInput, SerializedParam}; -use apollo_proc_macros::latency_histogram; +use apollo_proc_macros::{latency_histogram, sequencer_latency_histogram}; use body::events::EventIndex; use cairo_lang_starknet_classes::casm_contract_class::CasmContractClass; use db::db_stats::{DbTableStats, DbWholeStats}; @@ -151,6 +153,7 @@ use crate::db::{ RW, }; use crate::header::StorageBlockHeader; +use crate::metrics::STORAGE_COMMIT_LATENCY; use crate::mmap_file::MMapFileStats; use crate::state::data::IndexedDeprecatedContractClass; use crate::version::{VersionStorageReader, VersionStorageWriter}; @@ -490,7 +493,7 @@ pub struct StorageTxn<'env, Mode: TransactionKind> { impl StorageTxn<'_, RW> { /// Commits the changes made in the transaction to the storage. - #[latency_histogram("storage_commit_latency_seconds", false)] + #[sequencer_latency_histogram(STORAGE_COMMIT_LATENCY, false)] pub fn commit(self) -> StorageResult<()> { self.file_handlers.flush(); Ok(self.txn.commit()?) diff --git a/crates/apollo_storage/src/metrics.rs b/crates/apollo_storage/src/metrics.rs new file mode 100644 index 00000000000..da58a64b764 --- /dev/null +++ b/crates/apollo_storage/src/metrics.rs @@ -0,0 +1,14 @@ +use apollo_metrics::define_metrics; + +define_metrics!( + Storage => { + MetricHistogram { STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, "storage_append_thin_state_diff_latency_seconds", "Latency to append thin state diff in storage (secs)" }, + MetricHistogram { STORAGE_COMMIT_LATENCY, "storage_commit_latency_seconds", "Latency to commit changes in storage (secs)" }, + }, +); + +#[allow(dead_code)] +pub(crate) fn register_metrics() { + STORAGE_APPEND_THIN_STATE_DIFF_LATENCY.register(); + STORAGE_COMMIT_LATENCY.register(); +} diff --git a/crates/apollo_storage/src/state/mod.rs b/crates/apollo_storage/src/state/mod.rs index 151594d5aa2..2b2fd37ec16 100644 --- a/crates/apollo_storage/src/state/mod.rs +++ b/crates/apollo_storage/src/state/mod.rs @@ -56,7 +56,7 @@ mod state_test; use std::collections::HashSet; -use apollo_proc_macros::latency_histogram; +use apollo_proc_macros::{latency_histogram, sequencer_latency_histogram}; use cairo_lang_starknet_classes::casm_contract_class::CasmContractClass; use indexmap::IndexMap; use starknet_api::block::BlockNumber; @@ -71,6 +71,7 @@ use crate::db::table_types::{CommonPrefix, DbCursorTrait, SimpleTable, Table}; use crate::db::{DbTransaction, TableHandle, TransactionKind, RW}; #[cfg(feature = "document_calls")] use crate::document_calls::{add_query, StorageQuery}; +use crate::metrics::STORAGE_APPEND_THIN_STATE_DIFF_LATENCY; use crate::mmap_file::LocationInFile; use crate::state::data::IndexedDeprecatedContractClass; use crate::{ @@ -431,7 +432,7 @@ impl<'env, Mode: TransactionKind> StateReader<'env, Mode> { } impl StateStorageWriter for StorageTxn<'_, RW> { - #[latency_histogram("storage_append_thin_state_diff_latency_seconds", false)] + #[sequencer_latency_histogram(STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, false)] fn append_state_diff( self, block_number: BlockNumber,