enable support for eth beacon single method tests

- rename EthConsensus to EthBeacon for related classes
- enable support for eth beacon methods to be tested in single method test mode
- moved method classes to methods module
- fixed incorrect EthValidatorStatus, removed PENDING_ONGOING as it doesn't exist, added PENDING_QUEUED
- enable retry for fetching blocks from eth beacon endpoint
-

Author: erwin-wee

chainbench/main.py

@@ -11,17 +11,19 @@
 from click import Context, Parameter
 from locust import runners
 
-from chainbench.user import EvmMethods
+from chainbench.user.methods import (
+    all_method_classes,
+    all_methods,
+    get_subclass_functions,
+)
 from chainbench.util.cli import (
     ContextData,
     ensure_results_dir,
     get_base_path,
     get_master_command,
     get_profile_path,
     get_profiles,
-    get_subclass_methods,
     get_worker_command,
-    task_to_method,
 )
 from chainbench.util.monitor import monitors
 from chainbench.util.notify import NoopNotifier, Notifier
@@ -52,8 +54,7 @@ def cli(ctx: Context):
 
 def validate_method(ctx: Context, param: Parameter, value: str) -> str:
     if value is not None:
-        method_list = [task_to_method(task) for task in get_subclass_methods(EvmMethods)]
-        if value not in method_list:
+        if value not in all_methods.keys():
             raise click.BadParameter(
                 f"Method {value} is not supported. " f"Use 'chainbench list methods' to list all available methods."
             )
@@ -229,7 +230,7 @@ def start(
         profile_dir = get_base_path(__file__) / "profile"
 
     if method:
-        profile_path = get_base_path(__file__) / "tools" / "test_method.py"
+        profile_path = Path(all_methods[method])
     elif profile:
         profile_path = get_profile_path(profile_dir, profile)
     else:
@@ -472,12 +473,14 @@ def profiles(profile_dir: Path) -> None:
 
 
 @_list.command(
-    help="Lists all available evm methods.",
+    help="Lists all available methods.",
 )
 def methods() -> None:
-    task_list = get_subclass_methods(EvmMethods)
-    for task in task_list:
-        click.echo(task_to_method(task))
+    for method_class in all_method_classes:
+        click.echo(f"\nMethods for {method_class.__name__}:")
+        task_list = get_subclass_functions(method_class)
+        for task in task_list:
+            click.echo(f"- {method_class.task_to_method(task.name)}")
 
 
 if __name__ == "__main__":

chainbench/profile/ethereum/consensus.py

@@ -1,27 +1,27 @@
 """
-Ethereum Consensus profile.
+Ethereum Beacon profile.
 """
 from locust import constant_pacing
 
-from chainbench.user.ethereum import EthConsensusMethods
+from chainbench.user.methods.ethereum import EthBeaconMethods
 
 # mypy: ignore_errors
 
 
-class EthereumConsensusProfile(EthConsensusMethods):
+class EthereumBeaconProfile(EthBeaconMethods):
     wait_time = constant_pacing(1)
     tasks = {
-        EthConsensusMethods.eth_v1_beacon_states_validators_random_ids_task: 698,
-        EthConsensusMethods.eth_v1_beacon_states_validators_head_task: 23,
-        EthConsensusMethods.eth_v1_beacon_states_head_committees_random_epoch_task: 10,
-        EthConsensusMethods.eth_v1_beacon_states_validators_random_validator_status_task: 10,
-        EthConsensusMethods.eth_v1_beacon_states_random_state_id_finality_checkpoints_task: 10,
-        EthConsensusMethods.eth_v2_beacon_blocks_random_block_id_task: 8,
-        EthConsensusMethods.eth_v2_beacon_blocks_head_task: 6,
-        EthConsensusMethods.eth_v1_beacon_headers_head_task: 4,
-        EthConsensusMethods.eth_v1_config_spec_task: 3,
-        EthConsensusMethods.eth_v1_node_health_task: 2,
-        EthConsensusMethods.eth_v2_beacon_blocks_finalized_task: 1,
-        EthConsensusMethods.eth_v1_beacon_headers_random_block_id_task: 1,
-        EthConsensusMethods.eth_v1_node_version_task: 1,
+        EthBeaconMethods.eth_v1_beacon_states_validators_random_ids_task: 698,
+        EthBeaconMethods.eth_v1_beacon_states_validators_head_task: 23,
+        EthBeaconMethods.eth_v1_beacon_states_head_committees_random_epoch_task: 10,
+        EthBeaconMethods.eth_v1_beacon_states_validators_random_status_task: 10,
+        EthBeaconMethods.eth_v1_beacon_states_random_state_id_finality_checkpoints_task: 10,
+        EthBeaconMethods.eth_v2_beacon_blocks_random_block_id_task: 8,
+        EthBeaconMethods.eth_v2_beacon_blocks_head_task: 6,
+        EthBeaconMethods.eth_v1_beacon_headers_head_task: 4,
+        EthBeaconMethods.eth_v1_config_spec_task: 3,
+        EthBeaconMethods.eth_v1_node_health_task: 2,
+        EthBeaconMethods.eth_v2_beacon_blocks_finalized_task: 1,
+        EthBeaconMethods.eth_v1_beacon_headers_random_block_id_task: 1,
+        EthBeaconMethods.eth_v1_node_version_task: 1,
     }

chainbench/profile/evm/all.py

@@ -4,10 +4,10 @@
 
 from locust import constant_pacing
 
-from chainbench.user import EvmMethods
-from chainbench.util.cli import get_subclass_methods
+from chainbench.user.methods import EvmMethods
+from chainbench.user.methods.common import get_subclass_functions
 
 
 class EvmAllProfile(EvmMethods):
     wait_time = constant_pacing(1)
-    tasks = [EvmMethods.get_method(method) for method in get_subclass_methods(EvmMethods)]
+    tasks = [task.method for task in get_subclass_functions(EvmMethods)]

chainbench/profile/starknet/wallet.py

@@ -70,7 +70,7 @@ def simulate_transaction_task(self):
             name="simulate_transaction",
             method="starknet_simulateTransaction",
             params=self._simulate_transaction_params_factory(get_rng()),
-            url_postfix="/rpc/v0.3",
+            path="/rpc/v0.3",
         ),
 
     @task(1)

chainbench/test_data/blockchain.py

@@ -7,7 +7,7 @@
 from gevent.lock import Semaphore as GeventSemaphore
 from tenacity import retry, stop_after_attempt
 
-from chainbench.util.http import HttpClient, HttpErrorLevel
+from chainbench.util.http import HttpClient
 from chainbench.util.rng import RNG, get_rng
 
 logger = logging.getLogger(__name__)
@@ -124,8 +124,8 @@ def __init__(self) -> None:
         self._lock.acquire()
         self._logger.debug("Locked")
 
-    def init_http_client(self, host_url: str):
-        self._client = HttpClient(host_url, error_level=HttpErrorLevel.ServerError)
+    def init_http_client(self, host_url: str) -> None:
+        self._client = HttpClient(host_url)
         self._logger.debug("Host: %s", host_url)
 
     @property

chainbench/test_data/ethereum.py

@@ -4,13 +4,16 @@
 from argparse import Namespace
 from dataclasses import dataclass
 
+from tenacity import retry, stop_after_attempt
+
 from chainbench.test_data.blockchain import (
     Block,
     BlockNotFoundError,
     BlockNumber,
     BlockRange,
     TestData,
 )
+from chainbench.util.http import HttpClient, HttpErrorLevel, HttpStatusError
 from chainbench.util.rng import RNG
 
 logger = logging.getLogger(__name__)
@@ -20,15 +23,15 @@
 
 class EthValidatorStatus:
     ACTIVE = "active"
-    ACTIVE_ONGOING = "active_ongoing"
     ACTIVE_EXITING = "active_exiting"
+    ACTIVE_ONGOING = "active_ongoing"
     ACTIVE_SLASHED = "active_slashed"
-    PENDING = "pending"
-    PENDING_ONGOING = "pending_ongoing"
-    PENDING_INITIALIZED = "pending_initialized"
     EXITED = "exited"
-    EXITED_UNSLASHED = "exited_unslashed"
     EXITED_SLASHED = "exited_slashed"
+    EXITED_UNSLASHED = "exited_unslashed"
+    PENDING = "pending"
+    PENDING_INITIALIZED = "pending_initialized"
+    PENDING_QUEUED = "pending_queued"
     WITHDRAWAL = "withdrawal"
     WITHDRAWAL_DONE = "withdrawal_done"
     WITHDRAWAL_POSSIBLE = "withdrawal_possible"
@@ -55,7 +58,7 @@ def to_dict(self):
 
 
 @dataclass(frozen=True)
-class EthConsensusBlock(Block):
+class EthBeaconBlock(Block):
     epoch: int
     committees: list[EthCommittee]
 
@@ -77,7 +80,7 @@ def from_response(cls, slot: BlockNumber, data: dict[str, t.Any]):
             for validator in committee["validators"]:
                 validators.append(EthValidator(validator))
             committees.append(EthCommittee(committee["index"], validators))
-        return EthConsensusBlock(slot, epoch, committees)
+        return EthBeaconBlock(slot, epoch, committees)
 
     def get_random_committee(self, rng: RNG) -> EthCommittee:
         return rng.random.choice(self.committees)
@@ -94,29 +97,42 @@ def get_random_validator_indexes(self, count: int, rng: RNG) -> list[str]:
         return [validator.index for validator in committee.validators[start_index : start_index + count]]
 
 
-class EthConsensusTestData(TestData[EthConsensusBlock]):
+class EthBeaconTestData(TestData[EthBeaconBlock]):
+    def init_http_client(self, host_url: str):
+        self._client = HttpClient(host_url, error_level=HttpErrorLevel.ServerError)
+        self._logger.debug("Host: %s", host_url)
+
     def fetch_block_header(self, block_id: int | str) -> dict[str, t.Any]:
         block_response = self.client.get(f"/eth/v1/beacon/headers/{block_id}")
         if block_response.status_code == 404:
             raise BlockNotFoundError
+        elif block_response.status_code != 200:
+            raise HttpStatusError(
+                block_response.status_code, f"Failed to fetch block header: {block_response.status_message}"
+            )
         return block_response.json["data"]["header"]["message"]
 
-    def fetch_block(self, block_id: int | str) -> EthConsensusBlock:
+    def fetch_block(self, block_id: int | str) -> EthBeaconBlock:
         if isinstance(block_id, str):
             if (block_id := block_id.lower()) not in (
                 "head",
                 "genesis",
                 "finalized",
             ):
                 raise ValueError("Invalid block identifier")
-        slot = int(self.fetch_block_header(block_id)["slot"])
+        try:
+            slot = int(self.fetch_block_header(block_id)["slot"])
 
-        committees_response = self.client.get(f"/eth/v1/beacon/states/{block_id}/committees", params={"slot": slot})
-        return EthConsensusBlock.from_response(slot, committees_response.json)
+            committees_response = self.client.get(f"/eth/v1/beacon/states/{block_id}/committees", params={"slot": slot})
+            return EthBeaconBlock.from_response(slot, committees_response.json)
+        except BlockNotFoundError:
+            return self.fetch_latest_block()
 
-    def fetch_latest_block(self) -> EthConsensusBlock:
+    @retry(reraise=True, stop=stop_after_attempt(5))
+    def fetch_latest_block(self) -> EthBeaconBlock:
         return self.fetch_block("head")
 
+    @retry(reraise=True, stop=stop_after_attempt(5))
     def fetch_latest_block_number(self) -> BlockNumber:
         return int(self.fetch_block_header("head")["slot"])
 
@@ -129,7 +145,7 @@ def _get_start_and_end_blocks(self, parsed_options: Namespace) -> BlockRange:
         logger.info("Using blocks from %s to %s as test data", start_block_number, end_block_number)
         return BlockRange(start_block_number, end_block_number)
 
-    def get_block_from_data(self, data: dict[str, t.Any] | str) -> EthConsensusBlock:
+    def get_block_from_data(self, data: dict[str, t.Any] | str) -> EthBeaconBlock:
         def get_committee(committee: dict[str, t.Any]) -> EthCommittee:
             return EthCommittee(
                 index=committee["index"],
@@ -144,7 +160,7 @@ def get_committee(committee: dict[str, t.Any]) -> EthCommittee:
         slot = data_dict["block_number"]
         epoch = slot // 32
         committees = [get_committee(committee) for committee in data_dict["committees"]]
-        return EthConsensusBlock(slot, epoch, committees)
+        return EthBeaconBlock(slot, epoch, committees)
 
     def get_random_epoch(self, rng: RNG) -> Epoch:
         return self.get_random_block(rng).epoch

chainbench/tools/test_method.py

@@ -1,6 +1,6 @@
 from chainbench.util.event import setup_event_listeners
 
-from .evm import EvmMethods, EvmUser
+from .evm import EvmUser
 from .http import HttpUser, JsonRpcUser
 from .solana import SolanaUser
 from .starknet import StarkNetUser
@@ -16,5 +16,4 @@
     "JsonRpcUser",
     "SolanaUser",
     "StarkNetUser",
-    "EvmMethods",
 ]