Update from upstream

Author: slowli

.github/workflows/ci-zk-toolbox-reusable.yml

@@ -90,13 +90,30 @@ jobs:
 
       - name: Run server
         run: |
-          ci_run zk_inception server --ignore-prerequisites -a --verbose &>server.log &
+          ci_run zk_inception server --ignore-prerequisites &>server.log &
           ci_run sleep 5
 
       - name: Run integration tests
         run: |
           ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose
 
+
+      - name: Run external node server
+        run: |
+          ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \
+          --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545  
+          ci_run zk_inception external-node init --ignore-prerequisites
+          ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log &
+          ci_run sleep 5
+
+      - name: Run integration tests en
+        run: |
+          ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose --external-node
+
       - name: Show server.log logs
         if: always()
         run: ci_run cat server.log || true
+      - name: Show external_node.log logs
+        if: always()
+        run: ci_run cat external_node.log || true
+

bin/zkt

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+cd $(dirname $0)
+cd ../zk_toolbox
+
+cargo install --path ./crates/zk_inception --force
+cargo install --path ./crates/zk_supervisor --force

chains/era/ZkStack.yaml

@@ -4,6 +4,7 @@ chain_id: 271
 prover_version: NoProofs
 configs: ./chains/era/configs/
 rocks_db_path: ./chains/era/db/
+external_node_config_path: ./chains/era/configs/external_node
 l1_batch_commit_data_generator_mode: Rollup
 base_token:
   address: '0x0000000000000000000000000000000000000001'

core/bin/external_node/src/config/mod.rs

@@ -421,6 +421,9 @@ pub(crate) struct OptionalENConfig {
     #[serde(default = "OptionalENConfig::default_snapshots_recovery_postgres_max_concurrency")]
     pub snapshots_recovery_postgres_max_concurrency: NonZeroUsize,
 
+    #[serde(default)]
+    pub snapshots_recovery_object_store: Option<ObjectStoreConfig>,
+
     /// Enables pruning of the historical node state (Postgres and Merkle tree). The node will retain
     /// recent state and will continuously remove (prune) old enough parts of the state in the background.
     #[serde(default)]
@@ -619,6 +622,10 @@ impl OptionalENConfig {
                 .as_ref()
                 .map(|a| a.enabled)
                 .unwrap_or_default(),
+            snapshots_recovery_object_store: load_config!(
+                general_config.snapshot_recovery,
+                object_store
+            ),
             pruning_chunk_size: load_optional_config_or_default!(
                 general_config.pruning,
                 chunk_size,
@@ -798,9 +805,11 @@ impl OptionalENConfig {
     }
 
     fn from_env() -> anyhow::Result<Self> {
-        envy::prefixed("EN_")
+        let mut result: OptionalENConfig = envy::prefixed("EN_")
             .from_env()
-            .context("could not load external node config")
+            .context("could not load external node config")?;
+        result.snapshots_recovery_object_store = snapshot_recovery_object_store_config().ok();
+        Ok(result)
     }
 
     pub fn polling_interval(&self) -> Duration {

core/bin/external_node/src/config/observability.rs

@@ -26,6 +26,8 @@ pub(crate) struct ObservabilityENConfig {
     /// Log format to use: either `plain` (default) or `json`.
     #[serde(default)]
     pub log_format: LogFormat,
+    // Log directives in format that is used in `RUST_LOG`
+    pub log_directives: Option<String>,
 }
 
 impl ObservabilityENConfig {
@@ -80,6 +82,9 @@ impl ObservabilityENConfig {
 
     pub fn build_observability(&self) -> anyhow::Result<zksync_vlog::ObservabilityGuard> {
         let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(self.log_format);
+        if let Some(log_directives) = self.log_directives.clone() {
+            builder = builder.with_log_directives(log_directives)
+        };
         // Some legacy deployments use `unset` as an equivalent of `None`.
         let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset");
         if let Some(sentry_url) = sentry_url {
@@ -100,7 +105,7 @@ impl ObservabilityENConfig {
     }
 
     pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result<Self> {
-        let (sentry_url, sentry_environment, log_format) =
+        let (sentry_url, sentry_environment, log_format, log_directives) =
             if let Some(observability) = general_config.observability.as_ref() {
                 (
                     observability.sentry_url.clone(),
@@ -109,9 +114,10 @@ impl ObservabilityENConfig {
                         .log_format
                         .parse()
                         .context("Invalid log format")?,
+                    observability.log_directives.clone(),
                 )
             } else {
-                (None, None, LogFormat::default())
+                (None, None, LogFormat::default(), None)
             };
         let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) =
             if let Some(prometheus) = general_config.prometheus_config.as_ref() {
@@ -130,6 +136,7 @@ impl ObservabilityENConfig {
             sentry_url,
             sentry_environment,
             log_format,
+            log_directives,
         })
     }
 }

core/bin/external_node/src/init.rs

@@ -3,6 +3,7 @@
 use std::time::Instant;
 
 use anyhow::Context as _;
+use zksync_config::ObjectStoreConfig;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_health_check::AppHealthCheck;
 use zksync_node_sync::genesis::perform_genesis_if_needed;
@@ -12,13 +13,12 @@ use zksync_snapshots_applier::{SnapshotsApplierConfig, SnapshotsApplierTask};
 use zksync_types::{L1BatchNumber, L2ChainId};
 use zksync_web3_decl::client::{DynClient, L2};
 
-use crate::config::snapshot_recovery_object_store_config;
-
 #[derive(Debug)]
 pub(crate) struct SnapshotRecoveryConfig {
     /// If not specified, the latest snapshot will be used.
     pub snapshot_l1_batch_override: Option<L1BatchNumber>,
     pub drop_storage_key_preimages: bool,
+    pub object_store_config: Option<ObjectStoreConfig>,
 }
 
 #[derive(Debug)]
@@ -91,7 +91,9 @@ pub(crate) async fn ensure_storage_initialized(
             )?;
 
             tracing::warn!("Proceeding with snapshot recovery. This is an experimental feature; use at your own risk");
-            let object_store_config = snapshot_recovery_object_store_config()?;
+            let object_store_config = recovery_config.object_store_config.context(
+                "Snapshot object store must be presented if snapshot recovery is activated",
+            )?;
             let object_store = ObjectStoreFactory::new(object_store_config)
                 .create_store()
                 .await?;

core/bin/external_node/src/main.rs

@@ -974,6 +974,7 @@ async fn run_node(
                 drop_storage_key_preimages: config
                     .experimental
                     .snapshots_recovery_drop_storage_key_preimages,
+                object_store_config: config.optional.snapshots_recovery_object_store.clone(),
             });
     ensure_storage_initialized(
         connection_pool.clone(),

core/lib/basic_types/src/web3/mod.rs

@@ -827,21 +827,33 @@ pub enum TransactionCondition {
 }
 
 // `FeeHistory`: from `web3::types::fee_history`
+// Adapted to support blobs.
 
 /// The fee history type returned from `eth_feeHistory` call.
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct FeeHistory {
     /// Lowest number block of the returned range.
     pub oldest_block: BlockNumber,
-    /// A vector of block base fees per gas. This includes the next block after the newest of the returned range, because this value can be derived from the newest block. Zeroes are returned for pre-EIP-1559 blocks.
+    /// A vector of block base fees per gas. This includes the next block after the newest of the returned range,
+    /// because this value can be derived from the newest block. Zeroes are returned for pre-EIP-1559 blocks.
     #[serde(default)] // some node implementations skip empty lists
     pub base_fee_per_gas: Vec<U256>,
     /// A vector of block gas used ratios. These are calculated as the ratio of gas used and gas limit.
     #[serde(default)] // some node implementations skip empty lists
     pub gas_used_ratio: Vec<f64>,
-    /// A vector of effective priority fee per gas data points from a single block. All zeroes are returned if the block is empty. Returned only if requested.
+    /// A vector of effective priority fee per gas data points from a single block. All zeroes are returned if
+    /// the block is empty. Returned only if requested.
     pub reward: Option<Vec<Vec<U256>>>,
+    /// An array of base fees per blob gas for blocks. This includes the next block following the newest in the
+    /// returned range, as this value can be derived from the latest block. For blocks before EIP-4844, zeroes
+    /// are returned.
+    #[serde(default)] // some node implementations skip empty lists
+    pub base_fee_per_blob_gas: Vec<U256>,
+    /// An array showing the ratios of blob gas used in blocks. These ratios are calculated by dividing blobGasUsed
+    /// by the maximum blob gas per block.
+    #[serde(default)] // some node implementations skip empty lists
+    pub blob_gas_used_ratio: Vec<f64>,
 }
 
 // `SyncInfo`, `SyncState`: from `web3::types::sync_state`

core/lib/eth_client/src/clients/http/query.rs

@@ -8,7 +8,7 @@ use zksync_web3_decl::error::{ClientRpcContext, EnrichedClientError, EnrichedCli
 use super::{decl::L1EthNamespaceClient, Method, COUNTERS, LATENCIES};
 use crate::{
     types::{ExecutedTxStatus, FailureInfo},
-    EthInterface, RawTransactionBytes,
+    BaseFees, EthInterface, RawTransactionBytes,
 };
 
 #[async_trait]
@@ -78,7 +78,15 @@ where
         &self,
         upto_block: usize,
         block_count: usize,
-    ) -> EnrichedClientResult<Vec<u64>> {
+    ) -> EnrichedClientResult<Vec<BaseFees>> {
+        // Non-panicking conversion to u64.
+        fn cast_to_u64(value: U256, tag: &str) -> EnrichedClientResult<u64> {
+            u64::try_from(value).map_err(|_| {
+                let err = ClientError::Custom(format!("{tag} value does not fit in u64"));
+                EnrichedClientError::new(err, "cast_to_u64").with_arg("value", &value)
+            })
+        }
+
         const MAX_REQUEST_CHUNK: usize = 1024;
 
         COUNTERS.call[&(Method::BaseFeeHistory, self.component())].inc();
@@ -103,11 +111,34 @@ where
                 .with_arg("chunk_size", &chunk_size)
                 .with_arg("block", &chunk_end)
                 .await?;
-            history.extend(fee_history.base_fee_per_gas);
+
+            // Check that the lengths are the same.
+            // Per specification, the values should always be provided, and must be 0 for blocks
+            // prior to EIP-4844.
+            // https://ethereum.github.io/execution-apis/api-documentation/
+            if fee_history.base_fee_per_gas.len() != fee_history.base_fee_per_blob_gas.len() {
+                tracing::error!(
+                    "base_fee_per_gas and base_fee_per_blob_gas have different lengths: {} and {}",
+                    fee_history.base_fee_per_gas.len(),
+                    fee_history.base_fee_per_blob_gas.len()
+                );
+            }
+
+            for (base, blob) in fee_history
+                .base_fee_per_gas
+                .into_iter()
+                .zip(fee_history.base_fee_per_blob_gas)
+            {
+                let fees = BaseFees {
+                    base_fee_per_gas: cast_to_u64(base, "base_fee_per_gas")?,
+                    base_fee_per_blob_gas: blob,
+                };
+                history.push(fees)
+            }
         }
 
         latency.observe();
-        Ok(history.into_iter().map(|fee| fee.as_u64()).collect())
+        Ok(history)
     }
 
     async fn get_pending_block_base_fee_per_gas(&self) -> EnrichedClientResult<U256> {