apollo_storage: add panels

Author: ayeletstarkware

Cargo.lock

@@ -941,7 +941,28 @@
         "extra_params": {}
       }
     ],
-    "Storage": [],
+    "Storage": [
+      {
+        "title": "storage_append_thin_state_diff_latency_seconds",
+        "description": "Latency to append thin state diff in storage (secs)",
+        "type": "timeseries",
+        "exprs": [
+          "histogram_quantile(0.50, sum by (le) (rate(storage_append_thin_state_diff_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
+          "histogram_quantile(0.95, sum by (le) (rate(storage_append_thin_state_diff_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
+        ],
+        "extra_params": {}
+      },
+      {
+        "title": "storage_commit_latency_seconds",
+        "description": "Latency to commit changes in storage (secs)",
+        "type": "timeseries",
+        "exprs": [
+          "histogram_quantile(0.50, sum by (le) (rate(storage_commit_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))",
+          "histogram_quantile(0.95, sum by (le) (rate(storage_commit_latency_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[5m])))"
+        ],
+        "extra_params": {}
+      }
+    ],
     "MempoolP2p": [
       {
         "title": "apollo_mempool_p2p_num_connected_peers",
@@ -2537,4 +2558,4 @@
       }
     ]
   }
-}
+}

crates/apollo_dashboard/resources/dev_grafana.json

@@ -1,5 +1,17 @@
-use crate::dashboard::Row;
+use apollo_storage::metrics::{STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, STORAGE_COMMIT_LATENCY};
+
+use crate::dashboard::{Panel, PanelType, Row};
+
+fn get_storage_append_thin_state_diff_latency() -> Panel {
+    Panel::from_hist(STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, PanelType::TimeSeries)
+}
+fn get_storage_commit_latency() -> Panel {
+    Panel::from_hist(STORAGE_COMMIT_LATENCY, PanelType::TimeSeries)
+}
 
 pub(crate) fn get_storage_row() -> Row {
-    Row::new("Storage", vec![])
+    Row::new(
+        "Storage",
+        vec![get_storage_append_thin_state_diff_latency(), get_storage_commit_latency()],
+    )
 }

crates/apollo_dashboard/src/panels/storage.rs

@@ -17,6 +17,7 @@ required-features = ["clap", "statistical"]
 
 [dependencies]
 apollo_config.workspace = true
+apollo_metrics.workspace = true
 apollo_proc_macros.workspace = true
 byteorder.workspace = true
 cairo-lang-casm = { workspace = true, features = ["parity-scale-codec"] }

crates/apollo_storage/Cargo.toml

@@ -83,6 +83,8 @@ pub mod class_manager;
 pub mod compiled_class;
 #[cfg(feature = "document_calls")]
 pub mod document_calls;
+#[allow(missing_docs)]
+pub mod metrics;
 pub mod storage_metrics;
 // TODO(yair): Make the compression_utils module pub(crate) or extract it from the crate.
 #[doc(hidden)]
@@ -109,7 +111,7 @@ use std::sync::Arc;
 
 use apollo_config::dumping::{prepend_sub_config_name, ser_param, SerializeConfig};
 use apollo_config::{ParamPath, ParamPrivacyInput, SerializedParam};
-use apollo_proc_macros::latency_histogram;
+use apollo_proc_macros::{latency_histogram, sequencer_latency_histogram};
 use body::events::EventIndex;
 use cairo_lang_starknet_classes::casm_contract_class::CasmContractClass;
 use db::db_stats::{DbTableStats, DbWholeStats};
@@ -151,6 +153,7 @@ use crate::db::{
     RW,
 };
 use crate::header::StorageBlockHeader;
+use crate::metrics::STORAGE_COMMIT_LATENCY;
 use crate::mmap_file::MMapFileStats;
 use crate::state::data::IndexedDeprecatedContractClass;
 use crate::version::{VersionStorageReader, VersionStorageWriter};
@@ -490,7 +493,7 @@ pub struct StorageTxn<'env, Mode: TransactionKind> {
 
 impl StorageTxn<'_, RW> {
     /// Commits the changes made in the transaction to the storage.
-    #[latency_histogram("storage_commit_latency_seconds", false)]
+    #[sequencer_latency_histogram(STORAGE_COMMIT_LATENCY, false)]
     pub fn commit(self) -> StorageResult<()> {
         self.file_handlers.flush();
         Ok(self.txn.commit()?)

crates/apollo_storage/src/lib.rs

@@ -0,0 +1,14 @@
+use apollo_metrics::define_metrics;
+
+define_metrics!(
+    Storage => {
+        MetricHistogram { STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, "storage_append_thin_state_diff_latency_seconds", "Latency to append thin state diff in storage (secs)" },
+        MetricHistogram { STORAGE_COMMIT_LATENCY, "storage_commit_latency_seconds", "Latency to commit changes in storage (secs)" },
+    },
+);
+
+#[allow(dead_code)]
+pub(crate) fn register_metrics() {
+    STORAGE_APPEND_THIN_STATE_DIFF_LATENCY.register();
+    STORAGE_COMMIT_LATENCY.register();
+}

crates/apollo_storage/src/metrics.rs

@@ -56,7 +56,7 @@ mod state_test;
 
 use std::collections::HashSet;
 
-use apollo_proc_macros::latency_histogram;
+use apollo_proc_macros::{latency_histogram, sequencer_latency_histogram};
 use cairo_lang_starknet_classes::casm_contract_class::CasmContractClass;
 use indexmap::IndexMap;
 use starknet_api::block::BlockNumber;
@@ -71,6 +71,7 @@ use crate::db::table_types::{CommonPrefix, DbCursorTrait, SimpleTable, Table};
 use crate::db::{DbTransaction, TableHandle, TransactionKind, RW};
 #[cfg(feature = "document_calls")]
 use crate::document_calls::{add_query, StorageQuery};
+use crate::metrics::STORAGE_APPEND_THIN_STATE_DIFF_LATENCY;
 use crate::mmap_file::LocationInFile;
 use crate::state::data::IndexedDeprecatedContractClass;
 use crate::{
@@ -431,7 +432,7 @@ impl<'env, Mode: TransactionKind> StateReader<'env, Mode> {
 }
 
 impl StateStorageWriter for StorageTxn<'_, RW> {
-    #[latency_histogram("storage_append_thin_state_diff_latency_seconds", false)]
+    #[sequencer_latency_histogram(STORAGE_APPEND_THIN_STATE_DIFF_LATENCY, false)]
     fn append_state_diff(
         self,
         block_number: BlockNumber,