Merge pull request #116 from semiotic-ai/suchapalaver/fix-examples

build: port examples from beacon protos and fix

Author: suchapalaver

Cargo.lock

@@ -19,6 +19,7 @@ alloy-primitives = "1.4.0"
 anyhow = "1.0.100"
 arbitrum-ve = { path = "crates/arbitrum-ve" }
 base64 = "0.22.1"
+beacon-protos = { git = "https://github.com/semiotic-ai/beacon-protos.git", branch = "main" }
 bincode = { version = "2.0.1", features = ["serde"] }
 clap = { version = "4.5.23", features = ["derive"] }
 criterion = { version = "0.7.0", features = ["html_reports"] }
@@ -30,9 +31,11 @@ ethereum_ssz = "0.9.0"
 ethereum_ssz_derive = "0.9.0"
 firehose-protos = { path = "crates/firehose-protos" }
 firehose-rs = "0.3.0"
+futures = "0.3.31"
 header-accumulator = { path = "crates/header-accumulator" }
 hex = "0.4.3"
 keccak-hash = "0.11.0"
+kzg = { git = "https://github.com/sigp/lighthouse.git", branch = "stable" }
 merkle_proof = { git = "https://github.com/sigp/lighthouse.git", branch = "stable" }
 parquet = "56.2.0"
 primitive-types = "0.14.0"
@@ -58,6 +61,7 @@ tracing-subscriber = "0.3.19"
 tree_hash = "0.10.0"
 tree_hash_derive = "0.10.0"
 validation = { path = "crates/validation" }
+vee = { path = "crates/vee" }
 types = { git = "https://github.com/sigp/lighthouse.git", branch = "stable" }
 zstd = "0.13.3"
 

Cargo.toml

@@ -5,6 +5,10 @@ edition = "2021"
 
 [dev-dependencies]
 alloy-primitives.workspace = true
+beacon-protos.workspace = true
 firehose-client = { git = "https://github.com/semiotic-ai/firehose-client.git", branch = "main" }
 firehose-protos.workspace = true
+futures.workspace = true
+hex.workspace = true
 tokio.workspace = true
+vee.workspace = true

crates/firehose-protos-examples/Cargo.toml

@@ -0,0 +1,61 @@
+// Copyright 2024-, Semiotic AI, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! # Fetch Beacon Block
+//!
+//! Demonstrates how to fetch a single block from Beacon Firehose, using the `Fetch` API.
+
+use beacon_protos::{Block as BeaconBlock, Body};
+use firehose_client::{Chain, FirehoseClient};
+use vee::EthBlock;
+
+#[tokio::main]
+async fn main() {
+    // Show matching data from execution layer and beacon chain
+    let mut execution_layer_client = FirehoseClient::new(Chain::Ethereum);
+
+    let response = execution_layer_client
+        .fetch_block(20672593)
+        .await
+        .unwrap()
+        .unwrap();
+
+    let block = EthBlock::try_from(response.into_inner()).unwrap();
+
+    assert_eq!(block.number, 20672593);
+    assert_eq!(
+        format!("0x{}", hex::encode(block.hash)).as_str(),
+        "0xea48ba1c8e38ea586239e9c5ec62949ddd79404c6006c099bb02a8b22ddd18e4"
+    );
+
+    let mut beacon_client = FirehoseClient::new(Chain::Beacon);
+    // This is the slot number for the Beacon block we want to fetch, but right now
+    // we don't have a way to map the block number of the execution block to the slot number
+    // of the Beacon block.
+    let response = beacon_client.fetch_block(9881091).await.unwrap().unwrap();
+    let block = BeaconBlock::try_from(response.into_inner()).unwrap();
+
+    assert_eq!(block.slot, 9881091);
+
+    let body = block.body.as_ref().unwrap();
+
+    match body {
+        Body::Deneb(body) => {
+            let execution_payload = body.execution_payload.as_ref().unwrap();
+
+            let block_hash = &execution_payload.block_hash;
+
+            assert_eq!(
+                format!("0x{}", hex::encode(block_hash)).as_str(),
+                "0xea48ba1c8e38ea586239e9c5ec62949ddd79404c6006c099bb02a8b22ddd18e4"
+            );
+
+            let block_number = execution_payload.block_number;
+
+            assert_eq!(block_number, 20672593);
+        }
+        _ => unimplemented!(),
+    };
+
+    println!("fetch_beacon ran to completion!");
+}

crates/firehose-protos-examples/examples/fetch_beacon.rs

@@ -0,0 +1,24 @@
+// Copyright 2024-, Semiotic AI, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! # Fetch Ethereum Block
+//!
+//! Demonstrates how to fetch a single block from Ethereum firehose.
+
+use firehose_client::{Chain, FirehoseClient};
+use vee::EthBlock as Block;
+
+#[tokio::main]
+async fn main() {
+    let mut client = FirehoseClient::new(Chain::Ethereum);
+    let response = client.fetch_block(20672593).await.unwrap().unwrap();
+    let block = Block::try_from(response.into_inner()).unwrap();
+
+    assert_eq!(block.number, 20672593);
+    assert_eq!(
+        format!("0x{}", hex::encode(block.hash)).as_str(),
+        "0xea48ba1c8e38ea586239e9c5ec62949ddd79404c6006c099bb02a8b22ddd18e4"
+    );
+
+    println!("fetch_beacon completed successfully!");
+}

crates/firehose-protos-examples/examples/fetch_ethereum.rs

@@ -0,0 +1,35 @@
+// Copyright 2024-, Semiotic AI, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! # Example: Stream Beacon Blocks
+//!
+//! Demonstrates how to stream a range of blocks from Firehose Beacon
+
+use beacon_protos::Block as FirehoseBeaconBlock;
+use firehose_client::{Chain, FirehoseClient};
+use futures::StreamExt;
+
+#[tokio::main]
+async fn main() {
+    // Testing this so far without proper benchmarking, the time taken to fetch the blocks
+    // grows linearly with the number of TOTAL_BLOCKS requested, to around 20 minutes for 8192 blocks!
+    const TOTAL_SLOTS: u64 = 100;
+    const START_SLOT: u64 = 9968872;
+
+    let mut client = FirehoseClient::new(Chain::Beacon);
+    let mut stream = client
+        .stream_blocks::<FirehoseBeaconBlock>(START_SLOT, TOTAL_SLOTS)
+        .await
+        .unwrap();
+
+    let mut blocks: Vec<FirehoseBeaconBlock> = Vec::with_capacity(TOTAL_SLOTS as usize);
+
+    while let Some(block) = stream.next().await {
+        blocks.push(block);
+    }
+
+    // For now, just using this to signal that the test has completed
+    assert_eq!(blocks.len(), TOTAL_SLOTS as usize);
+
+    println!("stream_beacon ran to completion!");
+}

crates/firehose-protos-examples/examples/stream_beacon.rs

@@ -0,0 +1,34 @@
+// Copyright 2024-, Semiotic AI, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! # Example: Stream Ethereum Blocks
+//!
+//! This example demonstrates how to stream Ethereum blocks using the Firehose client.
+use firehose_client::{Chain, FirehoseClient};
+use futures::StreamExt;
+use vee::EthBlock as Block;
+
+#[tokio::main]
+async fn main() {
+    // Testing this so far without proper benchmarking, the time taken to fetch the blocks
+    // grows linearly with the number of TOTAL_BLOCKS requested, to around 20 minutes for 8192 blocks!
+    const TOTAL_BLOCKS: u64 = 100;
+    const START_BLOCK: u64 = 19581798;
+
+    let mut client = FirehoseClient::new(Chain::Ethereum);
+    let mut stream = client
+        .stream_blocks::<Block>(START_BLOCK, TOTAL_BLOCKS)
+        .await
+        .unwrap();
+
+    let mut blocks: Vec<Block> = Vec::with_capacity(TOTAL_BLOCKS as usize);
+
+    while let Some(block) = stream.next().await {
+        blocks.push(block);
+    }
+
+    // For now, just using this to signal that the test has completed
+    assert_eq!(blocks.len(), TOTAL_BLOCKS as usize);
+
+    println!("stream_ethereum ran successfully");
+}