feat: unify the metrics across the repo (#21)

Author: guorong009

.gitignore

@@ -18,3 +18,4 @@ dist/
 Dockerfile
 rust-project.json
 tmp/
+*.json

binius/src/bin/measure_lookup.rs

@@ -1,27 +1,38 @@
 use anyhow::Error;
-use binius::bench::{prove, sha256_with_lookup_prepare, verify};
-use utils::bench::measure_peak_memory;
+use binius::bench::{prove, sha256_with_lookup_prepare};
+use binius_utils::SerializeBytes;
+use utils::bench::{SubMetrics, measure_peak_memory, write_json_submetrics};
 
 fn main() -> Result<(), Error> {
+    let json_file = "sha2_binius_lookup_submetrics.json";
+
+    let input_num_bytes = 2048;
+    let metrics = benchmark_sha2(input_num_bytes)?;
+
+    write_json_submetrics(json_file, &metrics);
+
+    Ok(())
+}
+
+fn benchmark_sha2(num_bytes: usize) -> Result<SubMetrics, Error> {
+    let mut metrics = SubMetrics::new(num_bytes);
+
     let allocator = bumpalo::Bump::new();
 
     let ((constraint_system, args, witness, backend), peak_memory) =
         measure_peak_memory(|| sha256_with_lookup_prepare(&allocator));
+    metrics.preprocessing_peak_memory = peak_memory;
 
-    println!(
-        "Preprocessing(lookup) peak memory: {} MB",
-        peak_memory as f32 / (1024.0 * 1024.0),
-    );
+    let mut buffer: Vec<u8> = Vec::new();
+    let _ = constraint_system
+        .serialize(&mut buffer, binius_utils::SerializationMode::CanonicalTower)
+        .expect("Failed to serialize constraint system");
+    metrics.preprocessing_size = buffer.len();
 
-    let ((cs, args, proof), peak_memory) =
+    let ((_, _, proof), peak_memory) =
         measure_peak_memory(|| prove(constraint_system, args, witness, backend));
+    metrics.proving_peak_memory = peak_memory;
+    metrics.proof_size = proof.get_proof_size();
 
-    println!(
-        "Proving(lookup) peak memory: {} MB",
-        peak_memory as f32 / (1024.0 * 1024.0)
-    );
-
-    verify(cs, args, proof);
-
-    Ok(())
+    Ok(metrics)
 }

binius/src/bin/measure_no_lookup.rs

@@ -1,27 +1,38 @@
 use anyhow::Error;
-use binius::bench::{prove, sha256_no_lookup_prepare, verify};
-use utils::bench::measure_peak_memory;
+use binius::bench::{prove, sha256_no_lookup_prepare};
+use binius_utils::SerializeBytes;
+use utils::bench::{SubMetrics, measure_peak_memory, write_json_submetrics};
 
 fn main() -> Result<(), Error> {
+    let json_file = "sha2_binius_no_lookup_submetrics.json";
+
+    let input_num_bytes = 2048;
+    let metrics = benchmark_sha2(input_num_bytes)?;
+
+    write_json_submetrics(json_file, &metrics);
+
+    Ok(())
+}
+
+fn benchmark_sha2(num_bytes: usize) -> Result<SubMetrics, Error> {
+    let mut metrics = SubMetrics::new(num_bytes);
+
     let allocator = bumpalo::Bump::new();
 
     let ((constraint_system, args, witness, backend), peak_memory) =
         measure_peak_memory(|| sha256_no_lookup_prepare(&allocator));
+    metrics.preprocessing_peak_memory = peak_memory;
 
-    println!(
-        "Preprocessing(no lookup) peak memory: {} MB",
-        peak_memory as f32 / (1024.0 * 1024.0),
-    );
+    let mut buffer: Vec<u8> = Vec::new();
+    let _ = constraint_system
+        .serialize(&mut buffer, binius_utils::SerializationMode::CanonicalTower)
+        .expect("Failed to serialize constraint system");
+    metrics.preprocessing_size = buffer.len();
 
-    let ((cs, args, proof), peak_memory) =
+    let ((_, _, proof), peak_memory) =
         measure_peak_memory(|| prove(constraint_system, args, witness, backend));
+    metrics.proving_peak_memory = peak_memory;
+    metrics.proof_size = proof.get_proof_size();
 
-    println!(
-        "Proving(no lookup) peak memory: {} MB",
-        peak_memory as f32 / (1024.0 * 1024.0)
-    );
-
-    verify(cs, args, proof);
-
-    Ok(())
+    Ok(metrics)
 }

guests/src/ecdsa.rs

@@ -1,7 +1,7 @@
 use k256::{
-    ecdsa::{signature::Verifier, Signature, VerifyingKey},
-    elliptic_curve::sec1::EncodedPoint,
     Secp256k1,
+    ecdsa::{Signature, VerifyingKey, signature::Verifier},
+    elliptic_curve::sec1::EncodedPoint,
 };
 use serde::{Deserialize, Serialize};
 

plonky2/src/bin/measure.rs

@@ -2,18 +2,25 @@ use plonky2::{plonk::config::PoseidonGoldilocksConfig, util::serialization::Writ
 use plonky2_sha256::bench::{prove, sha256_no_lookup_prepare};
 use plonky2_u32::gates::arithmetic_u32::{U32GateSerializer, U32GeneratorSerializer};
 
-use utils::bench::measure_peak_memory;
+use utils::bench::{SubMetrics, measure_peak_memory, write_json_submetrics};
 
 const D: usize = 2;
 type C = PoseidonGoldilocksConfig;
 
 fn main() {
-    let ((data, pw), peak_memory) = measure_peak_memory(sha256_no_lookup_prepare);
+    let json_file = "sha2_plonky2_submetrics.json";
 
-    println!(
-        "Preprocessing peak memory: {} GB",
-        peak_memory as f32 / (1024.0 * 1024.0 * 1024.0),
-    );
+    let input_num_bytes = 2048;
+    let metrics = benchmark_sha2(input_num_bytes);
+
+    write_json_submetrics(json_file, &metrics);
+}
+
+fn benchmark_sha2(num_bytes: usize) -> SubMetrics {
+    let mut metrics = SubMetrics::new(num_bytes);
+
+    let ((data, pw), peak_memory) = measure_peak_memory(|| sha256_no_lookup_prepare());
+    metrics.preprocessing_peak_memory = peak_memory;
 
     let gate_serializer = U32GateSerializer;
     let common_data_size = data.common.to_bytes(&gate_serializer).unwrap().len();
@@ -28,8 +35,10 @@ fn main() {
         "Common data size: {}B, Prover data size: {}B",
         common_data_size, prover_data_size
     );
+    metrics.preprocessing_size = prover_data_size + common_data_size;
 
     let (proof, peak_memory) = measure_peak_memory(|| prove(&data.prover_data(), pw));
+    metrics.proving_peak_memory = peak_memory;
 
     println!(
         "Proving peak memory: {} GB",
@@ -39,4 +48,7 @@ fn main() {
     let mut buffer = Vec::new();
     buffer.write_proof(&proof.proof).unwrap();
     println!("Proof size: {} KB", buffer.len() as f32 / 1024.0);
+    metrics.proof_size = buffer.len();
+
+    metrics
 }

plonky3-powdr/src/bin/measure.rs

@@ -1,20 +1,39 @@
-use sha::bench::{prepare_pipeline, prove, verify};
-use utils::bench::measure_peak_memory;
+use sha::bench::{prepare_pipeline, prove};
+use utils::bench::{SubMetrics, measure_peak_memory, write_json_submetrics};
 
 fn main() {
-    let (mut pipeline, peak_memory) = measure_peak_memory(prepare_pipeline);
+    let json_file = "sha2_plonky3_powdr_submetrics.json";
 
+    let input_num_bytes = 2048;
+    let metrics = benchmark_sha2(input_num_bytes);
+
+    write_json_submetrics(json_file, &metrics);
+}
+
+fn benchmark_sha2(num_bytes: usize) -> SubMetrics {
+    let mut metrics = SubMetrics::new(num_bytes);
+
+    let (mut pipeline, peak_memory) = measure_peak_memory(|| prepare_pipeline());
+    metrics.preprocessing_peak_memory = peak_memory;
     println!(
         "Preprocessing peak memory: {} GB",
         peak_memory as f32 / (1024.0 * 1024.0 * 1024.0),
     );
 
+    // Load the proving key and constants from the files.
+    let pk_bytes = std::fs::read("powdr-target/pkey.bin").expect("Unable to read file");
+    let constants_bytes = std::fs::read("powdr-target/constants.bin").expect("Unable to read file");
+    let pil_bytes = std::fs::read("powdr-target/guest.pil").expect("Unable to read file");
+    metrics.preprocessing_size = pk_bytes.len() + constants_bytes.len() + pil_bytes.len();
+
     let (_, peak_memory) = measure_peak_memory(|| prove(&mut pipeline));
+    metrics.proving_peak_memory = peak_memory;
+    metrics.proof_size = pipeline.proof().unwrap().len();
 
     println!(
         "Proving peak memory: {} GB",
         peak_memory as f32 / (1024.0 * 1024.0 * 1024.0),
     );
 
-    verify(pipeline);
+    metrics
 }

plonky3-sp1/script/src/bin/measure.rs

@@ -1,25 +1,41 @@
 use sp1_sdk::{ProverClient, SP1Stdin, include_elf};
-use utils::bench::measure_peak_memory;
+use utils::bench::{SubMetrics, measure_peak_memory, write_json_submetrics};
 
 /// The ELF (executable and linkable format) file for the Succinct RISC-V zkVM.
 pub const SHA_ELF: &[u8] = include_elf!("sha-program");
 
 fn main() {
+    let json_file = "sha2_plonky3_sp1_submetrics.json";
+
+    let input_num_bytes = 2048;
+    let metrics = benchmark_sha2(input_num_bytes);
+
+    write_json_submetrics(json_file, &metrics);
+}
+
+fn benchmark_sha2(input_num_bytes: usize) -> SubMetrics {
+    let mut metrics = SubMetrics::new(input_num_bytes);
+
+    // Load the proving key and verifying key from the files.
+    let pk_bytes = std::fs::read("pk.bin").expect("Unable to read file");
+    let pk: sp1_sdk::SP1ProvingKey = bincode::deserialize(&pk_bytes).unwrap();
+    // Load the proof from the file.
+    let proof_bytes = std::fs::read("proof.bin").expect("Unable to read file");
+
     // Setup the prover client.
     let client = ProverClient::from_env();
     let stdin = SP1Stdin::new();
 
     // Setup the program for proving.
     let ((_, _), peak_memory) = measure_peak_memory(|| client.setup(SHA_ELF));
 
+    metrics.preprocessing_peak_memory = peak_memory;
     println!(
         "Preprocessing peak memory: {} GB",
         peak_memory as f32 / (1024.0 * 1024.0 * 1024.0)
     );
 
-    // Load the proving key and verifying key from the files.
-    let pk_bytes = std::fs::read("pk.bin").expect("Unable to read file");
-    let pk: sp1_sdk::SP1ProvingKey = bincode::deserialize(&pk_bytes).unwrap();
+    metrics.preprocessing_size = pk_bytes.len() + SHA_ELF.len();
 
     // Generate the proof
     let (_, peak_memory) = measure_peak_memory(|| {
@@ -28,9 +44,13 @@ fn main() {
             .run()
             .expect("failed to generate proof")
     });
+    metrics.proving_peak_memory = peak_memory;
+    metrics.proof_size = proof_bytes.len();
 
     println!(
         "Proving peak memory: {} GB",
         peak_memory as f32 / (1024.0 * 1024.0 * 1024.0),
     );
+
+    metrics
 }

utils/src/bench.rs

@@ -1,16 +1,16 @@
 use human_repr::{HumanCount, HumanDuration};
 use serde::Serialize;
-use serde_with::{serde_as, DurationNanoSeconds};
+use serde_with::{DurationNanoSeconds, serde_as};
 use std::{
     fmt::Display,
     sync::{
-        atomic::{AtomicBool, AtomicUsize, Ordering},
         Arc,
+        atomic::{AtomicBool, AtomicUsize, Ordering},
     },
     thread,
     time::Duration,
 };
-use tabled::{settings::Style, Table, Tabled};
+use tabled::{Table, Tabled, settings::Style};
 
 fn get_current_memory_usage() -> Result<usize, std::io::Error> {
     unsafe {
@@ -129,3 +129,35 @@ pub fn write_csv(out_path: &str, results: &[Metrics]) {
     table.with(Style::modern());
     println!("{table}");
 }
+
+#[serde_as]
+#[derive(Serialize, Tabled)]
+pub struct SubMetrics {
+    #[tabled(display_with = "display_bytes")]
+    pub input_size: usize,
+    #[tabled(display_with = "display_bytes")]
+    pub proof_size: usize,
+    #[tabled(display_with = "display_bytes")]
+    pub proving_peak_memory: usize,
+    #[tabled(display_with = "display_bytes")]
+    pub preprocessing_size: usize,
+    #[tabled(display_with = "display_bytes")]
+    pub preprocessing_peak_memory: usize,
+}
+
+impl SubMetrics {
+    pub fn new(size: usize) -> Self {
+        SubMetrics {
+            input_size: size,
+            proof_size: 0,
+            proving_peak_memory: 0,
+            preprocessing_size: 0,
+            preprocessing_peak_memory: 0,
+        }
+    }
+}
+
+pub fn write_json_submetrics(output_path: &str, metrics: &SubMetrics) {
+    let json = serde_json::to_string_pretty(metrics).unwrap();
+    std::fs::write(output_path, json).unwrap();
+}

utils/src/bin/sign_ecdsa.rs

@@ -1,5 +1,5 @@
 use k256::{
-    ecdsa::{signature::Signer, Signature, SigningKey},
+    ecdsa::{Signature, SigningKey, signature::Signer},
     elliptic_curve::rand_core::OsRng,
 };
 use std::{fs::File, io::Write};

utils/src/lib.rs

@@ -1,6 +1,6 @@
 use guests::ecdsa::EcdsaVerifyInput;
-use k256::{ecdsa::Signature, elliptic_curve::sec1::EncodedPoint, Secp256k1};
-use rand::{rngs::StdRng, RngCore, SeedableRng};
+use k256::{Secp256k1, ecdsa::Signature, elliptic_curve::sec1::EncodedPoint};
+use rand::{RngCore, SeedableRng, rngs::StdRng};
 use std::fs;
 use std::fs::File;
 use std::io::Write;