Skip to content

Commit

Permalink
Enable end-to-end integrations tests including prover->verifier workf…
Browse files Browse the repository at this point in the history
…low (Sovereign-Labs#842)

* Switch ordering of witness

* Feature gate native in stf runner

* WIP: Fix chain-state signature. Done except for borsh

* Fix all interfaces

* lint; clippy

* Move curr_hash to MockBlockHeader

* lint

* fix benches

* add missing borrow

* remove commented code

* fix feature gating. Fix demo-prover

* fmt demo prover

* lint

* implement stv

* Got to a failing test

* Tests pass

* lint

* lint

* fix tests/features

* fix prover

* lint
  • Loading branch information
preston-evans98 authored Sep 12, 2023
1 parent 5269d2b commit a1d6e7d
Show file tree
Hide file tree
Showing 41 changed files with 440 additions and 122 deletions.
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion adapters/celestia/src/da_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use crate::types::{ExtendedDataSquare, FilteredCelestiaBlock, Row, RpcNamespaced
use crate::utils::BoxError;
use crate::verifier::address::CelestiaAddress;
use crate::verifier::proofs::{CompletenessProof, CorrectnessProof};
use crate::verifier::{CelestiaSpec, RollupParams, PFB_NAMESPACE};
use crate::verifier::{CelestiaSpec, CelestiaVerifier, RollupParams, PFB_NAMESPACE};
use crate::{
parse_pfb_namespace, BlobWithSender, CelestiaHeader, CelestiaHeaderResponse,
DataAvailabilityHeader,
Expand Down Expand Up @@ -143,6 +143,8 @@ impl CelestiaService {
impl DaService for CelestiaService {
type Spec = CelestiaSpec;

type Verifier = CelestiaVerifier;

type FilteredBlock = FilteredCelestiaBlock;

type Error = BoxError;
Expand Down
2 changes: 1 addition & 1 deletion adapters/celestia/src/verifier/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ impl da::DaVerifier for CelestiaVerifier {
}

#[cfg_attr(all(target_os = "zkvm", feature = "bench"), cycle_tracker)]
fn verify_relevant_tx_list<H: Digest>(
fn verify_relevant_tx_list(
&self,
block_header: &<Self::Spec as DaSpec>::BlockHeader,
txs: &[<Self::Spec as DaSpec>::BlobTransaction],
Expand Down
81 changes: 76 additions & 5 deletions adapters/risc0/src/guest.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
#[cfg(not(target_os = "zkvm"))]
use std::ops::DerefMut;

#[cfg(target_os = "zkvm")]
use risc0_zkvm::guest::env;
#[cfg(not(target_os = "zkvm"))]
use risc0_zkvm::serde::{Deserializer, WordRead};
use sov_rollup_interface::zk::{Zkvm, ZkvmGuest};

use crate::Risc0MethodId;

pub struct Risc0Guest;

#[cfg(target_os = "zkvm")]
impl ZkvmGuest for Risc0Guest {
fn read_from_host<T: serde::de::DeserializeOwned>(&self) -> T {
Expand All @@ -17,14 +20,82 @@ impl ZkvmGuest for Risc0Guest {
}
}

#[cfg(not(target_os = "zkvm"))]
#[derive(Default)]
struct Hints {
values: Vec<u32>,
position: usize,
}

#[cfg(not(target_os = "zkvm"))]
impl Hints {
pub fn with_hints(hints: Vec<u32>) -> Self {
Hints {
values: hints,
position: 0,
}
}
pub fn remaining(&self) -> usize {
self.values.len() - self.position
}
}

#[cfg(not(target_os = "zkvm"))]
impl WordRead for Hints {
fn read_words(&mut self, words: &mut [u32]) -> risc0_zkvm::serde::Result<()> {
if self.remaining() < words.len() {
return Err(risc0_zkvm::serde::Error::DeserializeUnexpectedEnd);
}
words.copy_from_slice(&self.values[self.position..self.position + words.len()]);
self.position += words.len();
Ok(())
}

fn read_padded_bytes(&mut self, bytes: &mut [u8]) -> risc0_zkvm::serde::Result<()> {
let remaining_bytes: &[u8] = bytemuck::cast_slice(&self.values[self.position..]);
if bytes.len() > remaining_bytes.len() {
return Err(risc0_zkvm::serde::Error::DeserializeUnexpectedEnd);
}
bytes.copy_from_slice(&remaining_bytes[..bytes.len()]);
self.position += bytes.len() / std::mem::size_of::<u32>();
Ok(())
}
}

#[derive(Default)]
pub struct Risc0Guest {
#[cfg(not(target_os = "zkvm"))]
hints: std::sync::Mutex<Hints>,
#[cfg(not(target_os = "zkvm"))]
commits: std::sync::Mutex<Vec<u32>>,
}

impl Risc0Guest {
pub fn new() -> Self {
Self::default()
}

#[cfg(not(target_os = "zkvm"))]
pub fn with_hints(hints: Vec<u32>) -> Self {
Self {
hints: std::sync::Mutex::new(Hints::with_hints(hints)),
commits: Default::default(),
}
}
}

#[cfg(not(target_os = "zkvm"))]
impl ZkvmGuest for Risc0Guest {
fn read_from_host<T: serde::de::DeserializeOwned>(&self) -> T {
unimplemented!("This method should only be called in zkvm mode")
let mut hints = self.hints.lock().unwrap();
let mut hints = hints.deref_mut();
T::deserialize(&mut Deserializer::new(&mut hints)).unwrap()
}

fn commit<T: serde::Serialize>(&self, _item: &T) {
unimplemented!("This method should only be called in zkvm mode")
fn commit<T: serde::Serialize>(&self, item: &T) {
self.commits.lock().unwrap().extend_from_slice(
&risc0_zkvm::serde::to_vec(item).expect("Serialization to vec is infallible"),
);
}
}

Expand Down
56 changes: 31 additions & 25 deletions adapters/risc0/src/host.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::cell::RefCell;
use std::sync::Mutex;

use risc0_zkvm::receipt::Receipt;
use risc0_zkvm::serde::to_vec;
Expand All @@ -9,50 +9,50 @@ use sov_rollup_interface::zk::{Zkvm, ZkvmHost};
#[cfg(feature = "bench")]
use sov_zk_cycle_utils::{cycle_count_callback, get_syscall_name, get_syscall_name_cycles};

use crate::guest::Risc0Guest;
#[cfg(feature = "bench")]
use crate::metrics::metrics_callback;
use crate::Risc0MethodId;

pub struct Risc0Host<'a> {
env: RefCell<ExecutorEnvBuilder<'a>>,
env: Mutex<Vec<u32>>,
elf: &'a [u8],
}

impl<'a> Risc0Host<'a> {
#[cfg(not(feature = "bench"))]
pub fn new(elf: &'a [u8]) -> Self {
let default_env = ExecutorEnvBuilder::default();

Self {
env: RefCell::new(default_env),
elf,
}
}
#[cfg(not(feature = "bench"))]
fn add_benchmarking_callbacks(env: ExecutorEnvBuilder<'_>) -> ExecutorEnvBuilder<'_> {
env
}

#[cfg(feature = "bench")]
pub fn new(elf: &'a [u8]) -> Self {
let mut default_env = ExecutorEnvBuilder::default();
#[cfg(feature = "bench")]
fn add_benchmarking_callbacks(mut env: ExecutorEnvBuilder<'_>) -> ExecutorEnvBuilder<'_> {
let metrics_syscall_name = get_syscall_name();
env.io_callback(metrics_syscall_name, metrics_callback);

let metrics_syscall_name = get_syscall_name();
default_env.io_callback(metrics_syscall_name, metrics_callback);
let cycles_syscall_name = get_syscall_name_cycles();
env.io_callback(cycles_syscall_name, cycle_count_callback);

let cycles_syscall_name = get_syscall_name_cycles();
default_env.io_callback(cycles_syscall_name, cycle_count_callback);
env
}

impl<'a> Risc0Host<'a> {
pub fn new(elf: &'a [u8]) -> Self {
Self {
env: RefCell::new(default_env),
env: Default::default(),
elf,
}
}

/// Run a computation in the zkvm without generating a receipt.
/// This creates the "Session" trace without invoking the heavy cryptographic machinery.
pub fn run_without_proving(&mut self) -> anyhow::Result<Session> {
let env = self.env.borrow_mut().build()?;
let env = add_benchmarking_callbacks(ExecutorEnvBuilder::default())
.add_input(&self.env.lock().unwrap())
.build()
.unwrap();
let mut executor = LocalExecutor::from_elf(env, self.elf)?;
executor.run()
}

/// Run a computation in the zkvm and generate a receipt.
pub fn run(&mut self) -> anyhow::Result<SessionReceipt> {
let session = self.run_without_proving()?;
Expand All @@ -61,13 +61,19 @@ impl<'a> Risc0Host<'a> {
}

impl<'a> ZkvmHost for Risc0Host<'a> {
fn write_to_guest<T: serde::Serialize>(&self, item: T) {
fn add_hint<T: serde::Serialize>(&self, item: T) {
let serialized = to_vec(&item).expect("Serialization to vec is infallible");
self.env.borrow_mut().add_input(&serialized);
self.env.lock().unwrap().extend_from_slice(&serialized[..]);
}

type Guest = Risc0Guest;

fn simulate_with_hints(&mut self) -> Self::Guest {
Risc0Guest::with_hints(std::mem::take(&mut self.env.lock().unwrap()))
}
}

impl<'prover> Zkvm for Risc0Host<'prover> {
impl<'host> Zkvm for Risc0Host<'host> {
type CodeCommitment = Risc0MethodId;

type Error = anyhow::Error;
Expand Down
12 changes: 6 additions & 6 deletions examples/demo-prover/host/benches/prover_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -201,22 +201,22 @@ async fn main() -> Result<(), anyhow::Error> {
for height in 2..(bincoded_blocks.len() as u64) {
num_blocks += 1;
let mut host = Risc0Host::new(ROLLUP_ELF);
host.write_to_guest(prev_state_root);
host.add_hint(prev_state_root);
println!(
"Requesting data for height {} and prev_state_root 0x{}",
height,
hex::encode(prev_state_root)
);
let filtered_block = &bincoded_blocks[height as usize];
let _header_hash = hex::encode(filtered_block.header.header.hash());
host.write_to_guest(&filtered_block.header);
host.add_hint(&filtered_block.header);
let (mut blob_txs, inclusion_proof, completeness_proof) = da_service
.extract_relevant_txs_with_proof(&filtered_block)
.await;

host.write_to_guest(&inclusion_proof);
host.write_to_guest(&completeness_proof);
host.write_to_guest(&blob_txs);
host.add_hint(&inclusion_proof);
host.add_hint(&completeness_proof);
host.add_hint(&blob_txs);

if !blob_txs.is_empty() {
num_blobs += blob_txs.len();
Expand All @@ -232,7 +232,7 @@ async fn main() -> Result<(), anyhow::Error> {
}
// println!("{:?}",result.batch_receipts);

host.write_to_guest(&result.witness);
host.add_hint(&result.witness);

println!("Skipping prover at block {height} to capture cycle counts\n");
let _receipt = host
Expand Down
12 changes: 6 additions & 6 deletions examples/demo-prover/host/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,15 +92,15 @@ async fn main() -> Result<(), anyhow::Error> {
// prev_state_root is the root after applying the block at height-1
// This is necessary since we're proving that the current state root for the current height is
// result of applying the block against state with root prev_state_root
host.write_to_guest(prev_state_root);
host.add_hint(prev_state_root);
info!(
"Requesting data for height {} and prev_state_root 0x{}",
height,
hex::encode(prev_state_root)
);
let filtered_block = da_service.get_finalized_at(height).await?;
let header_hash = hex::encode(filtered_block.header.header.hash());
host.write_to_guest(&filtered_block.header);
host.add_hint(&filtered_block.header);
// When we get a block from DA, we also need to provide proofs of completeness and correctness
// https://github.com/Sovereign-Labs/sovereign-sdk/blob/nightly/rollup-interface/specs/interfaces/da.md#type-inclusionmultiproof
let (mut blobs, inclusion_proof, completeness_proof) = da_service
Expand All @@ -115,8 +115,8 @@ async fn main() -> Result<(), anyhow::Error> {
);

// The above proofs of correctness and completeness need to passed to the prover
host.write_to_guest(&inclusion_proof);
host.write_to_guest(&completeness_proof);
host.add_hint(&inclusion_proof);
host.add_hint(&completeness_proof);

let result = app.stf.apply_slot(
Default::default(),
Expand All @@ -127,11 +127,11 @@ async fn main() -> Result<(), anyhow::Error> {

// The extracted blobs need to be passed to the prover after execution.
// (Without executing, the host couldn't prune any data that turned out to be irrelevant to the guest)
host.write_to_guest(&blobs);
host.add_hint(&blobs);

// Witness contains the merkle paths to the state root so that the code inside the VM
// can access state values (Witness can also contain other hints and proofs)
host.write_to_guest(&result.witness);
host.add_hint(&result.witness);

// Run the actual prover to generate a receipt that can then be verified
if !skip_prover {
Expand Down
7 changes: 3 additions & 4 deletions examples/demo-prover/methods/guest/src/bin/rollup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use sov_celestia_adapter::verifier::address::CelestiaAddress;
use sov_celestia_adapter::verifier::{CelestiaSpec, CelestiaVerifier};
use sov_celestia_adapter::{BlobWithSender, CelestiaHeader};
use sov_risc0_adapter::guest::Risc0Guest;
use sov_rollup_interface::crypto::NoOpHasher;
use sov_rollup_interface::da::{BlockHeaderTrait, DaSpec, DaVerifier};
use sov_rollup_interface::stf::StateTransitionFunction;
use sov_rollup_interface::zk::{StateTransition, ZkvmGuest};
Expand All @@ -32,7 +31,7 @@ risc0_zkvm::guest::entry!(main);
// 6. Output (Da hash, start_root, end_root, event_root)
pub fn main() {
env::write(&"Start guest\n");
let guest = Risc0Guest;
let guest = Risc0Guest{};

#[cfg(feature = "bench")]
let start_cycles = env::get_cycle_count();
Expand All @@ -54,12 +53,12 @@ pub fn main() {
});

let validity_condition = verifier
.verify_relevant_tx_list::<NoOpHasher>(&header, &blobs, inclusion_proof, completeness_proof)
.verify_relevant_tx_list(&header, &blobs, inclusion_proof, completeness_proof)
.expect("Transaction list must be correct");
env::write(&"Relevant txs verified\n");

// Step 3: Apply blobs
let mut app = create_zk_app_template::<Risc0Guest, CelestiaSpec>(prev_state_root_hash);
let mut app = create_zk_app_template::<Risc0Guest, CelestiaSpec>();

let witness: ArrayWitness = guest.read_from_host();
env::write(&"Witness have been read\n");
Expand Down
1 change: 0 additions & 1 deletion examples/demo-rollup/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ ethers-signers = { workspace = true }
ethers = { workspace = true }
revm = { workspace = true }

sov-demo-rollup = { path = ".", features = ["experimental"] }

[features]
default = []
Expand Down
Loading

0 comments on commit a1d6e7d

Please sign in to comment.