improve profile discovery (#12)

* introduce tags for debug and trace methods

* rework profile discovery and add light and heavy profiles for EVM

* update readme

Author: ave2me

README.md

@@ -103,8 +103,32 @@ After the test is finished, the tool will automatically quit.
 - `--autoquit`: This flag tells the Chainbench tool to automatically quit after the test has finished. This is useful for running the benchmark in an automated environment where manual intervention is not desired.
 - `--help`: This flag displays the help message.
 
+By default, Chainbench uses `constant_pacing` wait time with a value of 10 seconds. 
+This means a load test with 100 users will have a theoretical maximum of 10 rps. This allows us to have a target rps in a test, 
+and the result rps lower than the target rps means performance deterioration.
+
 ### Profiles
-Profiles are located in the [`profile`](chainbench/profile) directory. For a tutorial on how to create custom profiles, please refer to [this document](docs/PROFILE.md).
+Default profiles are located in the [`profile`](chainbench/profile) directory. For a tutorial on how to create custom profiles, please refer to [this document](docs/PROFILE.md).
+
+You can also use the `--profile-dir` flag to specify a custom directory with profiles. For example:
+```shell
+chainbench start --profile-dir /path/to/profiles --profile my-profile --users 50 --workers 2 --test-time 12h --target https://node-url --headless --autoquit
+```
+This will run a load test using `/path/to/profiles/my-profile.py` profile.
+
+It's possible to group profiles into directories. For example, you can create a directory called `bsc` and put all the BSC profiles there. Then you can run a load test using the following command:
+```shell
+chainbench start --profile-dir /path/to/profiles --profile bsc.my-profile --users 50 --workers 2 --test-time 12h --target https://node-url --headless --autoquit
+```
+
+Chainbench will look for the profile in `/path/to/profiles/bsc/my-profile.py`. Currently, only one level of nesting is supported.
+
+There are built-in `evm.light` and `evm.heavy` profiles for EVM-compatible chains.
+
+Here's an example of how to run a load test for Ethereum using the `evm.light` profile:
+```shell
+chainbench start --profile evm.light --users 50 --workers 2 --test-time 12h --target https://node-url --headless --autoquit
+```
 
 ### Web UI Mode
 

chainbench/main.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shlex
 import subprocess
 import sys
@@ -9,6 +10,7 @@
 from chainbench.util.cli import (
     ContextData,
     ensure_results_dir,
+    get_base_path,
     get_master_command,
     get_profile_path,
     get_worker_command,
@@ -48,6 +50,9 @@ def cli(ctx: click.Context):
     help="Profile to run",
     show_default=True,
 )
+@click.option(
+    "--profile-dir", default=None, type=click.Path(), help="Profile directory"
+)
 @click.option("--host", default=MASTER_HOST, help="Host to run on", show_default=True)
 @click.option("--port", default=MASTER_PORT, help="Port to run on", show_default=True)
 @click.option(
@@ -76,6 +81,7 @@ def cli(ctx: click.Context):
 def start(
     ctx: click.Context,
     profile: str,
+    profile_dir: Path | None,
     host: str,
     port: int,
     workers: int,
@@ -102,7 +108,10 @@ def start(
         click.echo("Target is required when running in headless mode")
         sys.exit(1)
 
-    profile_path = get_profile_path(profile, __file__)
+    if not profile_dir:
+        profile_dir = get_base_path(__file__)
+
+    profile_path = get_profile_path(profile_dir, profile)
 
     if not profile_path.exists():
         click.echo(f"Profile file {profile_path} does not exist")
@@ -131,12 +140,17 @@ def start(
         workers=workers,
         headless=headless,
         target=target,
+        # TODO: Add support for tags in the CLI
+        exclude_tags=["trace", "debug"],
     )
     if headless:
         click.echo(f"Starting master in headless mode for {profile}")
     else:
         click.echo(f"Starting master for {profile}")
-    master_args = shlex.split(master_command)
+
+    is_posix = os.name == "posix"
+
+    master_args = shlex.split(master_command, posix=is_posix)
     master_process = subprocess.Popen(master_args)
     ctx.obj.master = master_process
     # Start the Locust workers
@@ -150,12 +164,13 @@ def start(
             target=target,
             worker_id=worker_id,
             log_level=log_level,
+            # TODO: Add support for tags in the CLI
+            exclude_tags=["trace", "debug"],
         )
-        worker_args = shlex.split(worker_command)
+        worker_args = shlex.split(worker_command, posix=is_posix)
         worker_process = subprocess.Popen(worker_args)
         ctx.obj.workers.append(worker_process)
         click.echo(f"Starting worker {worker_id + 1} for {profile}")
-        # Print out the URL to access the test
     if headless:
         click.echo(f"Running test in headless mode for {profile}")
         ctx.obj.notifier.notify(
@@ -164,13 +179,15 @@ def start(
             tags=["loudspeaker"],
         )
     else:
+        # Print out the URL to access the test
         click.echo(f"Run test: http://127.0.0.1:8089 {profile}")
 
     for process in ctx.obj.workers:
         process.wait()
 
     if autoquit:
-        click.echo("Quitting when test is finished")
+        ctx.obj.master.wait()
+        click.echo("Quitting...")
         ctx.obj.master.terminate()
 
     ctx.obj.notifier.notify(

chainbench/profile/ethereum.py

@@ -15,7 +15,7 @@
     "Others" : 12
 ```
 """
-from locust import task
+from locust import tag, task
 
 from chainbench.user.evm import EVMBenchUser
 
@@ -91,8 +91,8 @@ def get_logs_task(self):
             params=self._get_logs_params_factory(),
         ),
 
-    # TODO: introduce tags to make it possible to filter out unsupported methods
-    # @task(3)
+    @tag("debug")
+    @task(3)
     def trace_transaction_task(self):
         self.make_call(
             name="trace_transaction",

chainbench/profile/evm/heavy.py

@@ -0,0 +1,32 @@
+"""
+Ethereum profile (heavy mode).
+"""
+from locust import task
+
+from chainbench.user.evm import EVMBenchUser
+
+
+class EthereumHeavyProfile(EVMBenchUser):
+    @task
+    def trace_transaction_task(self):
+        self.make_call(
+            name="trace_transaction",
+            method="debug_traceTransaction",
+            params=[],
+        ),
+
+    @task
+    def block_task(self):
+        self.make_call(
+            name="block",
+            method="trace_block",
+            params=self._block_by_number_params_factory(),
+        ),
+
+    @task
+    def get_logs_task(self):
+        self.make_call(
+            name="get_logs",
+            method="eth_getLogs",
+            params=self._get_logs_params_factory(),
+        ),

chainbench/profile/evm/light.py

@@ -0,0 +1,64 @@
+"""
+Ethereum profile (light mode).
+"""
+from locust import task
+
+from chainbench.user.evm import EVMBenchUser
+
+
+class EthereumLightProfile(EVMBenchUser):
+    @task
+    def get_transaction_receipt_task(self):
+        self.make_call(
+            name="get_transaction_receipt",
+            method="eth_getTransactionReceipt",
+            params=self._transaction_by_hash_params_factory(),
+        ),
+
+    @task
+    def block_number_task(self):
+        self.make_call(
+            name="block_number",
+            method="eth_blockNumber",
+            params=[],
+        ),
+
+    @task
+    def get_balance_task(self):
+        self.make_call(
+            name="get_balance",
+            method="eth_getBalance",
+            params=self._get_balance_params_factory_latest(),
+        ),
+
+    @task
+    def chain_id_task(self):
+        self.make_call(
+            name="chain_id",
+            method="eth_chainId",
+            params=[],
+        ),
+
+    @task
+    def get_block_by_number_task(self):
+        self.make_call(
+            name="get_block_by_number",
+            method="eth_getBlockByNumber",
+            params=self._block_by_number_params_factory(),
+        ),
+
+    @task
+    def get_transaction_by_hash_task(self):
+        self.make_call(
+            name="get_transaction_by_hash",
+            method="eth_getTransactionByHash",
+            params=self._transaction_by_hash_params_factory(),
+        ),
+
+    @task
+    def client_version_task(self):
+        self.make_call(
+            name="client_version",
+            method="web3_clientVersion",
+            params=[],
+        ),

chainbench/profile/polygon.py

@@ -18,7 +18,7 @@
     "Others" : 9
 ```
 """
-from locust import task
+from locust import tag, task
 
 from chainbench.user.evm import EVMBenchUser
 
@@ -94,8 +94,8 @@ def get_balance_task(self):
             params=self._get_balance_params_factory_latest(),
         ),
 
-    # TODO: introduce tags to make it possible to filter out unsupported methods
-    # @task(2)
+    @tag("trace")
+    @task(2)
     def block_task(self):
         self.make_call(
             name="block",

chainbench/user/base.py

@@ -36,15 +36,15 @@ def on_stop(self):
 
     def check_fatal(self, response: RestResponseContextManager):
         if response.status_code == 401:
-            self.logger.critical(f"⛔️ Unauthorized request to {response.url}")
+            self.logger.critical(f"Unauthorized request to {response.url}")
         elif response.status_code == 404:
-            self.logger.critical(f"⛔️ Not found: {response.url}")
+            self.logger.critical(f"Not found: {response.url}")
         elif 500 <= response.status_code <= 599:
             self.logger.critical(
-                f"⛔️ Got internal server error when requesting {response.url}"
+                f"Got internal server error when requesting {response.url}"
             )
         elif 300 <= response.status_code <= 399:
-            self.logger.critical(f"⛔️ Redirect error: {response.url}")
+            self.logger.critical(f"Redirect error: {response.url}")
 
     def check_response(self, response: RestResponseContextManager):
         """Check the response for errors."""

chainbench/util/cli.py

@@ -6,10 +6,20 @@
 from chainbench.util.notify import NoopNotifier, Notifier
 
 
-def get_profile_path(profile: str, src_path: str | Path) -> Path:
-    """Get profile path."""
+def get_base_path(src_path: str | Path) -> Path:
+    """Get base path."""
     curr_path = Path(src_path).resolve()
-    profile_path = curr_path.parent / f"profile/{profile}.py"
+    base_path = curr_path.parent / "profile"
+    return base_path
+
+
+def get_profile_path(base_path: Path, profile: str) -> Path:
+    """Get profile path."""
+    subdir, _, profile = profile.rpartition(".")
+    if subdir:
+        profile_path = base_path / subdir / f"{profile}.py"
+    else:
+        profile_path = base_path / f"{profile}.py"
     return profile_path
 
 
@@ -42,6 +52,7 @@ def get_master_command(
     results_path: Path,
     target: str | None = None,
     headless: bool = False,
+    exclude_tags: list[str] | None = None,
 ) -> str:
     """Generate master command."""
     command = (
@@ -59,6 +70,9 @@ def get_master_command(
     if headless:
         command += " --headless"
 
+    if exclude_tags:
+        command += f" --exclude-tags {' '.join(exclude_tags)}"
+
     return command
 
 
@@ -71,6 +85,7 @@ def get_worker_command(
     target: str | None = None,
     headless: bool = False,
     worker_id: int = 0,
+    exclude_tags: list[str] | None = None,
 ) -> str:
     """Generate worker command."""
     command = (
@@ -84,6 +99,9 @@ def get_worker_command(
     if headless:
         command += " --headless"
 
+    if exclude_tags:
+        command += f" --exclude-tags {' '.join(exclude_tags)}"
+
     return command
 
 

chainbench/util/event.py

@@ -32,12 +32,12 @@ def on_test_stop(environment, **_kwargs):
     if not isinstance(runner, MasterRunner):
         # Print worker details to the log
         logger.info(
-            f"🏁 Worker[{runner.worker_index:02d}]: Tests completed in "
+            f"Worker[{runner.worker_index:02d}]: Tests completed in "
             f"{Timer.get_time_diff(runner.worker_index):>.3f} seconds"
         )
     else:
         # Print master details to the log
-        logger.info("🏁 Master: The test is stopped")
+        logger.info("Master: The test is stopped")
 
 
 # Listener for the init event
@@ -51,11 +51,11 @@ def on_init(environment, **_kwargs):
 
     if isinstance(environment.runner, MasterRunner):
         # Print master details to the log
-        logger.info("🤖 I'm a master. Running tests for %s", host_under_test)
+        logger.info("I'm a master. Running tests for %s", host_under_test)
 
     if isinstance(environment.runner, WorkerRunner):
         # Print worker details to the log
-        logger.info("🤖 I'm a worker. Running tests for %s", host_under_test)
+        logger.info("I'm a worker. Running tests for %s", host_under_test)
         logger.info("Initializing test data...")
         for user in environment.runner.user_classes:
             if not hasattr(user, "test_data"):