diff --git a/crates/core/machine/src/riscv/shape.rs b/crates/core/machine/src/riscv/shape.rs index 56f415fdb0..7ad6b8f8e9 100644 --- a/crates/core/machine/src/riscv/shape.rs +++ b/crates/core/machine/src/riscv/shape.rs @@ -79,7 +79,6 @@ impl CoreShapeConfig { }) .collect(); - tracing::info!("Found shape"); let shape = CoreShape { inner: shape? }; Some(shape) } @@ -297,8 +296,8 @@ impl Default for CoreShapeConfig { let included_shapes = vec![]; // Preprocessed chip heights. - let program_heights = vec![Some(16), Some(20), Some(21), Some(22)]; - let program_memory_heights = vec![Some(16), Some(20), Some(21), Some(22)]; + let program_heights = vec![Some(16), Some(19), Some(20), Some(21), Some(22)]; + let program_memory_heights = vec![Some(16), Some(19), Some(20), Some(21), Some(22)]; let allowed_preprocessed_log_heights = HashMap::from([ (RiscvAir::Program(ProgramChip::default()), program_heights), @@ -314,7 +313,7 @@ impl Default for CoreShapeConfig { let shift_right_heights = vec![None, Some(10), Some(16), Some(19)]; let shift_left_heights = vec![None, Some(10), Some(16), Some(19)]; let lt_heights = vec![None, Some(10), Some(16), Some(19)]; - let memory_local_heights = vec![Some(16), Some(20)]; + let memory_local_heights = vec![Some(16), Some(19)]; let syscall_heights = vec![None, Some(19)]; let short_allowed_log_heights = HashMap::from([ diff --git a/crates/core/machine/src/utils/prove.rs b/crates/core/machine/src/utils/prove.rs index 30460abc51..a797fcdf75 100644 --- a/crates/core/machine/src/utils/prove.rs +++ b/crates/core/machine/src/utils/prove.rs @@ -327,11 +327,6 @@ where // Create the challenger and observe the verifying key. let mut challenger = prover.config().challenger(); pk.observe_into(&mut challenger); - // challenger.observe(pk.preprocessed_commit()); - // challenger.observe(pk.pc_start()); - // for _ in 0..7 { - // challenger.observe(Val::::zero()); - // } // Spawn the phase 1 prover thread. let phase_1_prover_span = tracing::Span::current().clone(); @@ -508,9 +503,6 @@ where // Let another worker update the state. record_gen_sync.advance_turn(); - #[cfg(feature = "debug")] - all_records_tx.send(records.clone()).unwrap(); - // Fix the shape of the records. if let Some(shape_config) = shape_config { for record in records.iter_mut() { @@ -518,6 +510,9 @@ where } } + #[cfg(feature = "debug")] + all_records_tx.send(records.clone()).unwrap(); + // Generate the traces. let mut local_traces = Vec::new(); tracing::debug_span!("generate local traces", index).in_scope(|| { diff --git a/crates/prover/scripts/build_compress_vks.rs b/crates/prover/scripts/build_compress_vks.rs index 6528642d00..ca93fbb613 100644 --- a/crates/prover/scripts/build_compress_vks.rs +++ b/crates/prover/scripts/build_compress_vks.rs @@ -1,21 +1,26 @@ -use std::{fs::File, path::PathBuf}; +use std::path::PathBuf; use clap::Parser; -use p3_baby_bear::BabyBear; -use sp1_core_machine::{riscv::CoreShapeConfig, utils::setup_logger}; -use sp1_prover::{utils::get_all_vk_digests, InnerSC, REDUCE_BATCH_SIZE}; -use sp1_recursion_circuit_v2::merkle_tree::MerkleTree; -use sp1_recursion_core_v2::shape::RecursionShapeConfig; +use sp1_core_machine::utils::setup_logger; +use sp1_prover::{components::DefaultProverComponents, shapes::build_vk_map, REDUCE_BATCH_SIZE}; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Args { #[clap(short, long)] build_dir: PathBuf, - #[clap(short, long)] + #[clap(short, long, default_value_t = false)] dummy: bool, #[clap(short, long, default_value_t = REDUCE_BATCH_SIZE)] reduce_batch_size: usize, + #[clap(short, long, default_value_t = 1)] + num_compiler_workers: usize, + #[clap(short, long, default_value_t = 1)] + num_setup_workers: usize, + #[clap(short, long)] + start: Option, + #[clap(short, long)] + end: Option, } fn main() { @@ -24,32 +29,19 @@ fn main() { let reduce_batch_size = args.reduce_batch_size; let build_dir = args.build_dir; - - let core_shape_config = CoreShapeConfig::default(); - let recursion_shape_config = RecursionShapeConfig::default(); - - std::fs::create_dir_all(&build_dir).expect("failed to create build directory"); - - tracing::info!("building compress vk map"); - let vk_map = get_all_vk_digests(&core_shape_config, &recursion_shape_config, reduce_batch_size); - tracing::info!("compress vks generated, number of keys: {}", vk_map.len()); - - // Save the vk map to a file. - tracing::info!("saving vk map to file"); - let vk_map_path = build_dir.join("vk_map.bin"); - let mut vk_map_file = File::create(vk_map_path).unwrap(); - bincode::serialize_into(&mut vk_map_file, &vk_map).unwrap(); - tracing::info!("File saved successfully."); - - // Build a merkle tree from the vk map. - tracing::info!("building merkle tree"); - let (root, merkle_tree) = - MerkleTree::::commit(vk_map.keys().cloned().collect()); - - // Saving merkle tree data to file. - tracing::info!("saving merkle tree to file"); - let merkle_tree_path = build_dir.join("merkle_tree.bin"); - let mut merkle_tree_file = File::create(merkle_tree_path).unwrap(); - bincode::serialize_into(&mut merkle_tree_file, &(root, merkle_tree)).unwrap(); - tracing::info!("File saved successfully."); + let dummy = args.dummy; + let num_compiler_workers = args.num_compiler_workers; + let num_setup_workers = args.num_setup_workers; + let range_start = args.start; + let range_end = args.end; + + build_vk_map::( + build_dir, + reduce_batch_size, + dummy, + num_compiler_workers, + num_setup_workers, + range_start, + range_end, + ); } diff --git a/crates/prover/src/lib.rs b/crates/prover/src/lib.rs index c9d3edc185..850a06f52a 100644 --- a/crates/prover/src/lib.rs +++ b/crates/prover/src/lib.rs @@ -13,6 +13,7 @@ pub mod build; pub mod components; +pub mod shapes; pub mod types; pub mod utils; pub mod verify; @@ -74,8 +75,8 @@ use sp1_recursion_core_v2::{ use sp1_recursion_circuit_v2::{ hash::FieldHasher, machine::{ - SP1CompressRootVerifier, SP1CompressRootVerifierWithVKey, SP1CompressShape, - SP1CompressWithVKeyVerifier, SP1CompressWithVKeyWitnessValues, SP1CompressWitnessValues, + SP1CompressRootVerifier, SP1CompressRootVerifierWithVKey, SP1CompressWithVKeyVerifier, + SP1CompressWithVKeyWitnessValues, SP1CompressWithVkeyShape, SP1CompressWitnessValues, SP1DeferredVerifier, SP1DeferredWitnessValues, SP1MerkleProofWitnessValues, SP1RecursionShape, SP1RecursionWitnessValues, SP1RecursiveVerifier, }, @@ -133,7 +134,8 @@ pub struct SP1Prover { pub recursion_cache_misses: AtomicUsize, - pub compress_programs: Mutex>>>, + pub compress_programs: + Mutex>>>, pub compress_cache_misses: AtomicUsize, @@ -141,11 +143,13 @@ pub struct SP1Prover { pub allowed_vk_map: BTreeMap<>::Digest, usize>, - pub merkle_tree: MerkleTree, + pub vk_merkle_tree: MerkleTree, pub core_shape_config: Option>, pub recursion_shape_config: Option>>, + + pub vk_verification: bool, } impl SP1Prover { @@ -187,9 +191,6 @@ impl SP1Prover { ) .expect("PROVER_COMPRESS_CACHE_SIZE must be a non-zero usize"); - let core_shape_config = CoreShapeConfig::default(); - let recursion_shape_config = RecursionShapeConfig::default(); - let allowed_vk_map = bincode::deserialize(VK_MAP_BYTES).expect("failed to deserialize vk map"); @@ -199,12 +200,15 @@ impl SP1Prover { let core_shape_config = env::var("FIX_CORE_SHAPES") .map(|v| v.eq_ignore_ascii_case("true")) .unwrap_or(true) - .then_some(core_shape_config); + .then_some(CoreShapeConfig::default()); let recursion_shape_config = env::var("FIX_RECURSION_SHAPES") .map(|v| v.eq_ignore_ascii_case("true")) .unwrap_or(true) - .then_some(recursion_shape_config); + .then_some(RecursionShapeConfig::default()); + + let vk_verification = + env::var("VERIFY_VK").map(|v| v.eq_ignore_ascii_case("true")).unwrap_or(false); Self { core_prover, @@ -216,10 +220,11 @@ impl SP1Prover { compress_programs: Mutex::new(LruCache::new(compress_cache_size)), compress_cache_misses: AtomicUsize::new(0), root, - merkle_tree, + vk_merkle_tree: merkle_tree, allowed_vk_map, core_shape_config, recursion_shape_config, + vk_verification, } } @@ -309,12 +314,19 @@ impl SP1Prover { let mut cache = self.recursion_programs.lock().unwrap(); cache .get_or_insert(input.shape(), || { - tracing::info!("Proof shape: {:?}", input.shape()); let misses = self.recursion_cache_misses.fetch_add(1, Ordering::Relaxed); - tracing::debug!("Core cache miss, misses: {}", misses); + tracing::debug!("core cache miss, misses: {}", misses); // Get the operations. let builder_span = tracing::debug_span!("build recursion program").entered(); let mut builder = Builder::::default(); + + // // TODO: remove comment or make a test flag. + // let dummy_input = SP1RecursionWitnessValues::::dummy( + // self.core_prover.machine(), + // &input.shape(), + // ); + // let input = dummy_input.read(&mut builder); + let input = input.read(&mut builder); SP1RecursiveVerifier::verify(&mut builder, self.core_prover.machine(), input); let operations = builder.into_operations(); @@ -346,11 +358,19 @@ impl SP1Prover { // Get the operations. let builder_span = tracing::debug_span!("build compress program").entered(); let mut builder = Builder::::default(); + + // TODO: remove comment or make a test flag. + // let dummy_input = SP1CompressWithVKeyWitnessValues::::dummy( + // self.compress_prover.machine(), + // &input.shape(), + // ); + // let input = dummy_input.read(&mut builder); let input = input.read(&mut builder); SP1CompressWithVKeyVerifier::verify( &mut builder, self.compress_prover.machine(), input, + self.vk_verification, ); let operations = builder.into_operations(); builder_span.exit(); @@ -381,6 +401,7 @@ impl SP1Prover { &mut builder, self.compress_prover.machine(), input, + self.vk_verification, ); let operations = builder.into_operations(); builder_span.exit(); @@ -506,7 +527,7 @@ impl SP1Prover { vks_and_proofs, start_reconstruct_deferred_digest: deferred_digest, is_complete: false, - sp1_vk: vk.clone(), + sp1_vk_digest: vk.hash_babybear(), end_pc: Val::::zero(), end_shard: last_proof_pv.shard + BabyBear::one(), end_execution_shard: last_proof_pv.execution_shard, @@ -1117,7 +1138,7 @@ impl SP1Prover { let proofs = vk_indices .iter() .map(|index| { - let (_, proof) = MerkleTree::open(&self.merkle_tree, *index); + let (_, proof) = MerkleTree::open(&self.vk_merkle_tree, *index); proof }) .collect(); diff --git a/crates/prover/src/shapes.rs b/crates/prover/src/shapes.rs new file mode 100644 index 0000000000..4ccfbf3261 --- /dev/null +++ b/crates/prover/src/shapes.rs @@ -0,0 +1,232 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::File, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use p3_baby_bear::BabyBear; +use p3_field::AbstractField; +use sp1_core_machine::riscv::CoreShapeConfig; +use sp1_recursion_circuit_v2::{ + machine::{ + SP1CompressWithVKeyWitnessValues, SP1CompressWithVkeyShape, SP1DeferredShape, + SP1DeferredWitnessValues, SP1RecursionShape, SP1RecursionWitnessValues, + }, + merkle_tree::MerkleTree, +}; +use sp1_recursion_core_v2::{shape::RecursionShapeConfig, RecursionProgram}; +use sp1_stark::{MachineProver, ProofShape, DIGEST_SIZE}; + +use crate::{ + components::SP1ProverComponents, utils::MaybeTakeIterator, CompressAir, HashableKey, InnerSC, + SP1Prover, +}; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SP1ProofShape { + Recursion(ProofShape), + Compress(Vec), + Deferred(ProofShape), +} + +pub enum SP1CompressProgramShape { + Recursion(SP1RecursionShape), + Compress(SP1CompressWithVkeyShape), + Deferred(SP1DeferredShape), +} + +pub fn build_vk_map( + build_dir: PathBuf, + reduce_batch_size: usize, + dummy: bool, + num_compiler_workers: usize, + num_setup_workers: usize, + range_start: Option, + range_end: Option, +) { + std::fs::create_dir_all(&build_dir).expect("failed to create build directory"); + + let prover = SP1Prover::::new(); + let core_shape_config = prover.core_shape_config.as_ref().expect("core shape config not found"); + let recursion_shape_config = + prover.recursion_shape_config.as_ref().expect("recursion shape config not found"); + + tracing::info!("building compress vk map"); + let vk_map = if dummy { + tracing::warn!("Making a dummy vk map"); + SP1ProofShape::dummy_vk_map(core_shape_config, recursion_shape_config, reduce_batch_size) + } else { + let (vk_tx, vk_rx) = std::sync::mpsc::channel(); + let (shape_tx, shape_rx) = std::sync::mpsc::sync_channel(num_compiler_workers); + let (program_tx, program_rx) = std::sync::mpsc::sync_channel(num_setup_workers); + + let shape_rx = Mutex::new(shape_rx); + let program_rx = Mutex::new(program_rx); + + let length = range_end.and_then(|end| end.checked_sub(range_start.unwrap_or(0))); + let generate_shapes = || { + SP1ProofShape::generate(core_shape_config, recursion_shape_config, reduce_batch_size) + .maybe_skip(range_start) + .maybe_take(length) + }; + + let num_shapes = generate_shapes().count(); + let height = num_shapes.next_power_of_two().ilog2() as usize; + + tracing::info!("There are {} shapes to generate", num_shapes); + + std::thread::scope(|s| { + // Initialize compiler workers. + for _ in 0..num_compiler_workers { + let program_tx = program_tx.clone(); + let shape_rx = &shape_rx; + let prover = &prover; + s.spawn(move || { + while let Ok(shape) = shape_rx.lock().unwrap().recv() { + let program = prover.program_from_shape(shape); + program_tx.send(program).unwrap(); + } + }); + } + + // Initialize setup workers. + for _ in 0..num_setup_workers { + let vk_tx = vk_tx.clone(); + let program_rx = &program_rx; + let prover = &prover; + s.spawn(move || { + while let Ok(program) = program_rx.lock().unwrap().recv() { + let (_, vk) = prover.compress_prover.setup(&program); + let vk_digest = vk.hash_babybear(); + vk_tx.send(vk_digest).unwrap(); + } + }); + } + + // Generate shapes and send them to the compiler workers. + generate_shapes() + .map(|shape| SP1CompressProgramShape::from_proof_shape(shape, height)) + .for_each(|program_shape| { + shape_tx.send(program_shape).unwrap(); + }); + + drop(shape_tx); + drop(program_tx); + drop(vk_tx); + + let mut vk_set = BTreeSet::new(); + + for vk_digest in vk_rx.iter().take(num_shapes) { + vk_set.insert(vk_digest); + } + + vk_set.into_iter().enumerate().map(|(i, vk_digest)| (vk_digest, i)).collect() + }) + }; + tracing::info!("compress vks generated, number of keys: {}", vk_map.len()); + + // Save the vk map to a file. + tracing::info!("saving vk map to file"); + let vk_map_path = build_dir.join("vk_map.bin"); + let mut vk_map_file = File::create(vk_map_path).unwrap(); + bincode::serialize_into(&mut vk_map_file, &vk_map).unwrap(); + tracing::info!("File saved successfully."); + + // Build a merkle tree from the vk map. + tracing::info!("building merkle tree"); + let (root, merkle_tree) = + MerkleTree::::commit(vk_map.keys().cloned().collect()); + + // Saving merkle tree data to file. + tracing::info!("saving merkle tree to file"); + let merkle_tree_path = build_dir.join("merkle_tree.bin"); + let mut merkle_tree_file = File::create(merkle_tree_path).unwrap(); + bincode::serialize_into(&mut merkle_tree_file, &(root, merkle_tree)).unwrap(); + tracing::info!("File saved successfully."); +} + +impl SP1ProofShape { + pub fn generate<'a>( + core_shape_config: &'a CoreShapeConfig, + recursion_shape_config: &'a RecursionShapeConfig>, + reduce_batch_size: usize, + ) -> impl Iterator + 'a { + core_shape_config + .generate_all_allowed_shapes() + .map(Self::Recursion) + .chain( + recursion_shape_config + .get_all_shape_combinations(1) + .map(|mut x| Self::Deferred(x.pop().unwrap())), + ) + .chain( + recursion_shape_config + .get_all_shape_combinations(reduce_batch_size) + .map(Self::Compress), + ) + } + + pub fn dummy_vk_map<'a>( + core_shape_config: &'a CoreShapeConfig, + recursion_shape_config: &'a RecursionShapeConfig>, + reduce_batch_size: usize, + ) -> BTreeMap<[BabyBear; DIGEST_SIZE], usize> { + Self::generate(core_shape_config, recursion_shape_config, reduce_batch_size) + .enumerate() + .map(|(i, _)| ([BabyBear::from_canonical_usize(i); DIGEST_SIZE], i)) + .collect() + } +} + +impl SP1CompressProgramShape { + pub fn from_proof_shape(shape: SP1ProofShape, height: usize) -> Self { + match shape { + SP1ProofShape::Recursion(proof_shape) => Self::Recursion(proof_shape.into()), + SP1ProofShape::Deferred(proof_shape) => Self::Deferred(proof_shape.into()), + SP1ProofShape::Compress(proof_shapes) => Self::Compress(SP1CompressWithVkeyShape { + compress_shape: proof_shapes.into(), + merkle_tree_height: height, + }), + } + } +} + +impl SP1Prover { + pub fn program_from_shape( + &self, + shape: SP1CompressProgramShape, + ) -> Arc> { + match shape { + SP1CompressProgramShape::Recursion(shape) => { + let input = SP1RecursionWitnessValues::dummy(self.core_prover.machine(), &shape); + self.recursion_program(&input) + } + SP1CompressProgramShape::Deferred(shape) => { + let input = SP1DeferredWitnessValues::dummy(self.compress_prover.machine(), &shape); + self.deferred_program(&input) + } + SP1CompressProgramShape::Compress(shape) => { + let input = + SP1CompressWithVKeyWitnessValues::dummy(self.compress_prover.machine(), &shape); + self.compress_program(&input) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_all_shapes() { + let core_shape_config = CoreShapeConfig::default(); + let recursion_shape_config = RecursionShapeConfig::default(); + let reduce_batch_size = 2; + let num_compress_shapes = + SP1ProofShape::generate(&core_shape_config, &recursion_shape_config, reduce_batch_size) + .count(); + println!("Number of compress shapes: {}", num_compress_shapes); + } +} diff --git a/crates/prover/src/utils.rs b/crates/prover/src/utils.rs index 4ac85e0428..4ad036e5c2 100644 --- a/crates/prover/src/utils.rs +++ b/crates/prover/src/utils.rs @@ -1,21 +1,19 @@ use std::{ borrow::Borrow, - collections::BTreeMap, fs::{self, File}, io::Read, + iter::{Skip, Take}, }; use p3_baby_bear::BabyBear; use p3_bn254_fr::Bn254Fr; use p3_field::{AbstractField, PrimeField32}; use sp1_core_executor::{Executor, Program}; -use sp1_core_machine::{io::SP1Stdin, reduce::SP1ReduceProof, riscv::CoreShapeConfig}; -use sp1_recursion_core_v2::{ - air::RecursionPublicValues, shape::RecursionShapeConfig, stark::config::BabyBearPoseidon2Outer, -}; +use sp1_core_machine::{io::SP1Stdin, reduce::SP1ReduceProof}; +use sp1_recursion_core_v2::{air::RecursionPublicValues, stark::config::BabyBearPoseidon2Outer}; use sp1_stark::{SP1CoreOpts, Word}; -use crate::{CompressAir, SP1CoreProofData}; +use crate::SP1CoreProofData; /// Get the SP1 vkey BabyBear Poseidon2 digest this reduce proof is representing. pub fn sp1_vkey_digest_babybear(proof: &SP1ReduceProof) -> [BabyBear; 8] { @@ -38,31 +36,6 @@ pub fn sp1_commited_values_digest_bn254(proof: &SP1ReduceProof, - recursion_shape_config: &RecursionShapeConfig>, - reduce_batch_size: usize, -) -> BTreeMap<[BabyBear; 8], usize> { - let mut vk_map = core_shape_config - .generate_all_allowed_shapes() - .enumerate() - .map(|(i, _)| ([BabyBear::from_canonical_usize(i); 8], i)) - .collect::>(); - - let num_first_layer_vks = vk_map.len(); - - vk_map.extend( - recursion_shape_config.get_all_shape_combinations(reduce_batch_size).enumerate().map( - |(i, _)| { - let index = num_first_layer_vks + i; - ([BabyBear::from_canonical_usize(index); 8], index) - }, - ), - ); - - vk_map -} - impl SP1CoreProofData { pub fn save(&self, path: &str) -> Result<(), std::io::Error> { let data = serde_json::to_string(self).unwrap(); @@ -131,18 +104,46 @@ pub fn words_to_bytes_be(words: &[u32; 8]) -> [u8; 32] { bytes } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_get_all_vk_digests() { - let core_shape_config = CoreShapeConfig::default(); - let recursion_shape_config = RecursionShapeConfig::default(); - let reduce_batch_size = 2; - let vk_digests = - get_all_vk_digests(&core_shape_config, &recursion_shape_config, reduce_batch_size); - println!("Number of vk digests: {}", vk_digests.len()); - assert!(vk_digests.len() < 1 << 24); +pub trait MaybeTakeIterator: Iterator { + fn maybe_skip(self, bound: Option) -> RangedIterator + where + Self: Sized, + { + match bound { + Some(bound) => RangedIterator::Skip(self.skip(bound)), + None => RangedIterator::Unbounded(self), + } + } + + fn maybe_take(self, bound: Option) -> RangedIterator + where + Self: Sized, + { + match bound { + Some(bound) => RangedIterator::Take(self.take(bound)), + None => RangedIterator::Unbounded(self), + } + } +} + +impl MaybeTakeIterator for I {} + +pub enum RangedIterator { + Unbounded(I), + Skip(Skip), + Take(Take), + Range(Take>), +} + +impl Iterator for RangedIterator { + type Item = I::Item; + + fn next(&mut self) -> Option { + match self { + RangedIterator::Unbounded(unbounded) => unbounded.next(), + RangedIterator::Skip(skip) => skip.next(), + RangedIterator::Take(take) => take.next(), + RangedIterator::Range(range) => range.next(), + } } } diff --git a/crates/recursion/circuit-v2/src/challenger.rs b/crates/recursion/circuit-v2/src/challenger.rs index 7bf491fcd2..fcfc729ea1 100644 --- a/crates/recursion/circuit-v2/src/challenger.rs +++ b/crates/recursion/circuit-v2/src/challenger.rs @@ -58,7 +58,7 @@ pub trait CanSampleBitsVariable { } /// Reference: [p3_challenger::DuplexChallenger] -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct DuplexChallengerVariable { pub sponge_state: [Felt; PERMUTATION_WIDTH], pub input_buffer: Vec>, diff --git a/crates/recursion/circuit-v2/src/machine/compress.rs b/crates/recursion/circuit-v2/src/machine/compress.rs index e6bc7e0e14..87644ed194 100644 --- a/crates/recursion/circuit-v2/src/machine/compress.rs +++ b/crates/recursion/circuit-v2/src/machine/compress.rs @@ -23,6 +23,7 @@ use sp1_recursion_core_v2::{ use sp1_stark::{ air::{MachineAir, POSEIDON_NUM_WORDS, PV_DIGEST_NUM_WORDS}, + baby_bear_poseidon2::BabyBearPoseidon2, ProofShape, ShardProof, StarkGenericConfig, StarkMachine, StarkVerifyingKey, Word, DIGEST_SIZE, }; @@ -30,7 +31,7 @@ use crate::{ challenger::CanObserveVariable, constraints::RecursiveVerifierConstraintFolder, machine::assert_complete, - stark::{ShardProofVariable, StarkVerifier}, + stark::{dummy_vk_and_shard_proof, ShardProofVariable, StarkVerifier}, utils::uninit_challenger_pv, BabyBearFriConfig, BabyBearFriConfigVariable, CircuitConfig, VerifyingKeyVariable, }; @@ -57,7 +58,7 @@ pub struct SP1CompressWitnessValues { pub is_complete: bool, } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct SP1CompressShape { proof_shapes: Vec, } @@ -477,3 +478,27 @@ impl SP1CompressWitnessValues { SP1CompressShape { proof_shapes } } } + +impl SP1CompressWitnessValues { + pub fn dummy>( + machine: &StarkMachine, + shape: &SP1CompressShape, + ) -> Self { + let vks_and_proofs = shape + .proof_shapes + .iter() + .map(|proof_shape| { + let (vk, proof) = dummy_vk_and_shard_proof(machine, proof_shape); + (vk, proof) + }) + .collect(); + + Self { vks_and_proofs, is_complete: false } + } +} + +impl From> for SP1CompressShape { + fn from(proof_shapes: Vec) -> Self { + Self { proof_shapes } + } +} diff --git a/crates/recursion/circuit-v2/src/machine/core.rs b/crates/recursion/circuit-v2/src/machine/core.rs index 21473f3d08..ced5090b47 100644 --- a/crates/recursion/circuit-v2/src/machine/core.rs +++ b/crates/recursion/circuit-v2/src/machine/core.rs @@ -19,6 +19,7 @@ use sp1_core_machine::{ use sp1_recursion_core_v2::air::PV_DIGEST_NUM_WORDS; use sp1_stark::{ air::{PublicValues, POSEIDON_NUM_WORDS}, + baby_bear_poseidon2::BabyBearPoseidon2, ProofShape, StarkMachine, Word, }; @@ -36,7 +37,7 @@ use sp1_recursion_core_v2::{ use crate::{ challenger::{CanObserveVariable, DuplexChallengerVariable, FieldChallengerVariable}, - stark::{ShardProofVariable, StarkVerifier}, + stark::{dummy_challenger, dummy_vk_and_shard_proof, ShardProofVariable, StarkVerifier}, BabyBearFriConfig, BabyBearFriConfigVariable, CircuitConfig, VerifyingKeyVariable, }; @@ -582,3 +583,28 @@ impl SP1RecursionWitnessValues { SP1RecursionShape { proof_shapes, is_complete: self.is_complete } } } + +impl SP1RecursionWitnessValues { + pub fn dummy( + machine: &StarkMachine>, + shape: &SP1RecursionShape, + ) -> Self { + let (mut vks, shard_proofs): (Vec<_>, Vec<_>) = + shape.proof_shapes.iter().map(|shape| dummy_vk_and_shard_proof(machine, shape)).unzip(); + let vk = vks.pop().unwrap(); + Self { + vk, + shard_proofs, + leaf_challenger: dummy_challenger(machine.config()), + initial_reconstruct_challenger: dummy_challenger(machine.config()), + is_complete: shape.is_complete, + is_first_shard: false, + } + } +} + +impl From for SP1RecursionShape { + fn from(proof_shape: ProofShape) -> Self { + Self { proof_shapes: vec![proof_shape], is_complete: false } + } +} diff --git a/crates/recursion/circuit-v2/src/machine/deferred.rs b/crates/recursion/circuit-v2/src/machine/deferred.rs index 5669f888f4..2d9d927aed 100644 --- a/crates/recursion/circuit-v2/src/machine/deferred.rs +++ b/crates/recursion/circuit-v2/src/machine/deferred.rs @@ -14,7 +14,8 @@ use sp1_recursion_compiler::ir::{Builder, Ext, Felt}; use sp1_stark::{ air::{MachineAir, POSEIDON_NUM_WORDS}, - ShardProof, StarkGenericConfig, StarkMachine, StarkVerifyingKey, Word, + baby_bear_poseidon2::BabyBearPoseidon2, + ProofShape, ShardProof, StarkMachine, StarkVerifyingKey, Word, }; use sp1_recursion_core_v2::{ @@ -25,18 +26,24 @@ use sp1_recursion_core_v2::{ use crate::{ challenger::{CanObserveVariable, DuplexChallengerVariable}, constraints::RecursiveVerifierConstraintFolder, - stark::{ShardProofVariable, StarkVerifier}, - BabyBearFriConfigVariable, CircuitConfig, VerifyingKeyVariable, + stark::{dummy_challenger, ShardProofVariable, StarkVerifier}, + BabyBearFriConfig, BabyBearFriConfigVariable, CircuitConfig, VerifyingKeyVariable, }; +use super::{SP1CompressShape, SP1CompressWitnessValues}; + pub struct SP1DeferredVerifier { _phantom: std::marker::PhantomData<(C, SC, A)>, } -pub struct SP1DeferredWitnessValues { +pub struct SP1DeferredShape { + inner: SP1CompressShape, +} + +pub struct SP1DeferredWitnessValues { pub vks_and_proofs: Vec<(StarkVerifyingKey, ShardProof)>, pub start_reconstruct_deferred_digest: [SC::Val; POSEIDON_NUM_WORDS], - pub sp1_vk: StarkVerifyingKey, + pub sp1_vk_digest: [SC::Val; DIGEST_SIZE], pub leaf_challenger: SC::Challenger, pub committed_value_digest: [Word; PV_DIGEST_NUM_WORDS], pub deferred_proofs_digest: [SC::Val; POSEIDON_NUM_WORDS], @@ -54,7 +61,7 @@ pub struct SP1DeferredWitnessVariable< > { pub vks_and_proofs: Vec<(VerifyingKeyVariable, ShardProofVariable)>, pub start_reconstruct_deferred_digest: [Felt; POSEIDON_NUM_WORDS], - pub sp1_vk: VerifyingKeyVariable, + pub sp1_vk_digest: [Felt; DIGEST_SIZE], pub leaf_challenger: SC::FriChallengerVariable, pub committed_value_digest: [Word>; PV_DIGEST_NUM_WORDS], pub deferred_proofs_digest: [Felt; POSEIDON_NUM_WORDS], @@ -94,7 +101,7 @@ where let SP1DeferredWitnessVariable { vks_and_proofs, start_reconstruct_deferred_digest, - sp1_vk, + sp1_vk_digest, leaf_challenger, committed_value_digest, deferred_proofs_digest, @@ -189,7 +196,7 @@ where deferred_public_values.last_finalize_addr_bits = finalize_addr_bits; // Set the sp1_vk_digest to be the hitned value. - deferred_public_values.sp1_vk_digest = sp1_vk.hash(builder); + deferred_public_values.sp1_vk_digest = sp1_vk_digest; // Set the committed value digest to be the hitned value. deferred_public_values.committed_value_digest = committed_value_digest; @@ -221,3 +228,35 @@ where SC::commit_recursion_public_values(builder, *deferred_public_values); } } + +impl SP1DeferredWitnessValues { + pub fn dummy>( + machine: &StarkMachine, + shape: &SP1DeferredShape, + ) -> Self { + let inner_witness = + SP1CompressWitnessValues::::dummy(machine, &shape.inner); + let vks_and_proofs = inner_witness.vks_and_proofs; + + Self { + vks_and_proofs, + leaf_challenger: dummy_challenger(machine.config()), + is_complete: true, + sp1_vk_digest: [BabyBear::zero(); DIGEST_SIZE], + start_reconstruct_deferred_digest: [BabyBear::zero(); POSEIDON_NUM_WORDS], + committed_value_digest: [Word::default(); PV_DIGEST_NUM_WORDS], + deferred_proofs_digest: [BabyBear::zero(); POSEIDON_NUM_WORDS], + end_pc: BabyBear::zero(), + end_shard: BabyBear::zero(), + end_execution_shard: BabyBear::zero(), + init_addr_bits: [BabyBear::zero(); 32], + finalize_addr_bits: [BabyBear::zero(); 32], + } + } +} + +impl From for SP1DeferredShape { + fn from(proof_shape: ProofShape) -> Self { + Self { inner: SP1CompressShape::from(vec![proof_shape]) } + } +} diff --git a/crates/recursion/circuit-v2/src/machine/root.rs b/crates/recursion/circuit-v2/src/machine/root.rs index 0d92efda84..b55fda4d19 100644 --- a/crates/recursion/circuit-v2/src/machine/root.rs +++ b/crates/recursion/circuit-v2/src/machine/root.rs @@ -74,10 +74,11 @@ where builder: &mut Builder, machine: &StarkMachine, input: SP1CompressWithVKeyWitnessVariable, + value_assertions: bool, ) { // Assert that the program is complete. builder.assert_felt_eq(input.compress_var.is_complete, C::F::one()); // Verify the proof, as a compress proof. - SP1CompressWithVKeyVerifier::verify(builder, machine, input); + SP1CompressWithVKeyVerifier::verify(builder, machine, input, value_assertions); } } diff --git a/crates/recursion/circuit-v2/src/machine/vkey_proof.rs b/crates/recursion/circuit-v2/src/machine/vkey_proof.rs index 489a26351c..a7e8cd1255 100644 --- a/crates/recursion/circuit-v2/src/machine/vkey_proof.rs +++ b/crates/recursion/circuit-v2/src/machine/vkey_proof.rs @@ -4,11 +4,13 @@ use itertools::Itertools; use p3_air::Air; use p3_baby_bear::BabyBear; use p3_commit::Mmcs; +use p3_field::AbstractField; use p3_matrix::dense::RowMajorMatrix; use sp1_recursion_compiler::ir::{Builder, Felt}; use sp1_recursion_core_v2::DIGEST_SIZE; use sp1_stark::{ - air::MachineAir, Com, InnerChallenge, OpeningProof, StarkGenericConfig, StarkMachine, + air::MachineAir, baby_bear_poseidon2::BabyBearPoseidon2, Com, InnerChallenge, OpeningProof, + StarkGenericConfig, StarkMachine, }; use crate::{ @@ -31,6 +33,13 @@ pub struct SP1MerkleProofVerifier { _phantom: PhantomData<(C, SC)>, } +/// The shape of the compress proof with vk validation proofs. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SP1CompressWithVkeyShape { + pub compress_shape: SP1CompressShape, + pub merkle_tree_height: usize, +} + /// Witness layout for the compress stage verifier. pub struct SP1MerkleProofWitnessVariable< C: CircuitConfig, @@ -62,14 +71,18 @@ where builder: &mut Builder, digests: Vec, input: SP1MerkleProofWitnessVariable, + value_assertions: bool, ) { let SP1MerkleProofWitnessVariable { vk_merkle_proofs, values, root } = input; - for ((proof, value), _expected_value) in + for ((proof, value), expected_value) in vk_merkle_proofs.into_iter().zip(values).zip(digests) { verify(builder, proof, value, root); - // TODO: comment back in. - // SC::assert_digest_eq(builder, expected_value, value); + if value_assertions { + SC::assert_digest_eq(builder, expected_value, value); + } else { + SC::assert_digest_eq(builder, value, value); + } } } } @@ -110,17 +123,46 @@ where builder: &mut Builder, machine: &StarkMachine, input: SP1CompressWithVKeyWitnessVariable, + value_assertions: bool, ) { let values = input.compress_var.vks_and_proofs.iter().map(|(vk, _)| vk.hash(builder)).collect_vec(); - SP1MerkleProofVerifier::verify(builder, values, input.merkle_var); + SP1MerkleProofVerifier::verify(builder, values, input.merkle_var, value_assertions); SP1CompressVerifier::verify(builder, machine, input.compress_var); } } impl> SP1CompressWithVKeyWitnessValues { - pub fn shape(&self) -> SP1CompressShape { - self.compress_val.shape() + pub fn shape(&self) -> SP1CompressWithVkeyShape { + let merkle_tree_height = self.merkle_val.vk_merkle_proofs.first().unwrap().path.len(); + SP1CompressWithVkeyShape { compress_shape: self.compress_val.shape(), merkle_tree_height } + } +} + +impl SP1MerkleProofWitnessValues { + pub fn dummy(num_proofs: usize, height: usize) -> Self { + let dummy_digest = [BabyBear::zero(); DIGEST_SIZE]; + let vk_merkle_proofs = + vec![MerkleProof { index: 0, path: vec![dummy_digest; height] }; num_proofs]; + let values = vec![dummy_digest; num_proofs]; + + Self { vk_merkle_proofs, values, root: dummy_digest } + } +} + +impl SP1CompressWithVKeyWitnessValues { + pub fn dummy>( + machine: &StarkMachine, + shape: &SP1CompressWithVkeyShape, + ) -> Self { + let compress_val = + SP1CompressWitnessValues::::dummy(machine, &shape.compress_shape); + let num_proofs = compress_val.vks_and_proofs.len(); + let merkle_val = SP1MerkleProofWitnessValues::::dummy( + num_proofs, + shape.merkle_tree_height, + ); + Self { compress_val, merkle_val } } } diff --git a/crates/recursion/circuit-v2/src/machine/witness.rs b/crates/recursion/circuit-v2/src/machine/witness.rs index ebf7e613d3..ac4b4f58da 100644 --- a/crates/recursion/circuit-v2/src/machine/witness.rs +++ b/crates/recursion/circuit-v2/src/machine/witness.rs @@ -164,7 +164,7 @@ where let vks_and_proofs = self.vks_and_proofs.read(builder); let start_reconstruct_deferred_digest = self.start_reconstruct_deferred_digest.read(builder); - let sp1_vk = self.sp1_vk.read(builder); + let sp1_vk_digest = self.sp1_vk_digest.read(builder); let leaf_challenger = self.leaf_challenger.read(builder); let committed_value_digest = self.committed_value_digest.read(builder); let deferred_proofs_digest = self.deferred_proofs_digest.read(builder); @@ -178,7 +178,7 @@ where SP1DeferredWitnessVariable { vks_and_proofs, start_reconstruct_deferred_digest, - sp1_vk, + sp1_vk_digest, leaf_challenger, committed_value_digest, deferred_proofs_digest, @@ -194,7 +194,7 @@ where fn write(&self, witness: &mut impl WitnessWriter) { self.vks_and_proofs.write(witness); self.start_reconstruct_deferred_digest.write(witness); - self.sp1_vk.write(witness); + self.sp1_vk_digest.write(witness); self.leaf_challenger.write(witness); self.committed_value_digest.write(witness); self.deferred_proofs_digest.write(witness); diff --git a/crates/recursion/circuit-v2/src/stark.rs b/crates/recursion/circuit-v2/src/stark.rs index 9306005dc5..eff2df0795 100644 --- a/crates/recursion/circuit-v2/src/stark.rs +++ b/crates/recursion/circuit-v2/src/stark.rs @@ -7,7 +7,7 @@ use p3_air::{Air, BaseAir}; use p3_baby_bear::BabyBear; use p3_commit::{Mmcs, Pcs, PolynomialSpace, TwoAdicMultiplicativeCoset}; use p3_field::{AbstractField, ExtensionField, Field, TwoAdicField}; -use p3_matrix::dense::RowMajorMatrix; +use p3_matrix::{dense::RowMajorMatrix, Dimensions}; use sp1_recursion_compiler::{ circuit::CircuitV2Builder, @@ -15,9 +15,9 @@ use sp1_recursion_compiler::{ prelude::Felt, }; use sp1_stark::{ - air::InteractionScope, baby_bear_poseidon2::BabyBearPoseidon2, AirOpenedValues, Chip, - ChipOpenedValues, InnerChallenge, ProofShape, ShardCommitment, ShardOpenedValues, ShardProof, - Val, PROOF_MAX_NUM_PVS, + air::InteractionScope, baby_bear_poseidon2::BabyBearPoseidon2, AirOpenedValues, Challenger, + Chip, ChipOpenedValues, InnerChallenge, ProofShape, ShardCommitment, ShardOpenedValues, + ShardProof, Val, PROOF_MAX_NUM_PVS, }; use sp1_stark::{air::MachineAir, StarkGenericConfig, StarkMachine, StarkVerifyingKey}; @@ -45,12 +45,19 @@ pub struct ShardProofVariable, SC: BabyBearFriConf pub public_values: Vec>, } +/// Get a dummy duplex challenger for use in dummy proofs. +pub fn dummy_challenger(config: &BabyBearPoseidon2) -> Challenger { + let mut challenger = config.challenger(); + challenger.input_buffer = vec![]; + challenger.output_buffer = vec![BabyBear::zero(); challenger.sponge_state.len()]; + challenger +} + /// Make a dummy shard proof for a given proof shape. -pub fn dummy_shard_proof>( +pub fn dummy_vk_and_shard_proof>( machine: &StarkMachine, - vk: &StarkVerifyingKey, shape: &ProofShape, -) -> ShardProof { +) -> (StarkVerifyingKey, ShardProof) { // Make a dummy commitment. let commitment = ShardCommitment { global_main_commit: dummy_hash(), @@ -79,24 +86,28 @@ pub fn dummy_shard_proof>( .collect(), }; + let mut preprocessed_names_and_dimensions = vec![]; let mut preprocessed_batch_shape = vec![]; let mut global_main_batch_shape = vec![]; let mut local_main_batch_shape = vec![]; let mut permutation_batch_shape = vec![]; let mut quotient_batch_shape = vec![]; - for info in vk.chip_information.iter() { - let name = &info.0; - let i = chip_ordering[name]; - let opened_values = &opened_values.chips[i]; - let prep_shape = PolynomialShape { - width: opened_values.preprocessed.local.len(), - log_degree: opened_values.log_degree, - }; - preprocessed_batch_shape.push(prep_shape); - } - - for (chip_opening, scope) in opened_values.chips.iter().zip_eq(chip_scopes.iter()) { + for ((chip, chip_opening), scope) in + shard_chips.iter().zip_eq(opened_values.chips.iter()).zip_eq(chip_scopes.iter()) + { + if !chip_opening.preprocessed.local.is_empty() { + let prep_shape = PolynomialShape { + width: chip_opening.preprocessed.local.len(), + log_degree: chip_opening.log_degree, + }; + preprocessed_names_and_dimensions.push(( + chip.name(), + prep_shape.width, + prep_shape.log_degree, + )); + preprocessed_batch_shape.push(prep_shape); + } let main_shape = PolynomialShape { width: chip_opening.main.local.len(), log_degree: chip_opening.log_degree, @@ -142,14 +153,43 @@ pub fn dummy_shard_proof>( let public_values = (0..PROOF_MAX_NUM_PVS).map(|_| BabyBear::zero()).collect::>(); - ShardProof { + // Get the preprocessed chip information. + let pcs = machine.config().pcs(); + let preprocessed_chip_information: Vec<_> = preprocessed_names_and_dimensions + .iter() + .map(|(name, width, log_height)| { + let domain = <::Pcs as Pcs< + ::Challenge, + ::Challenger, + >>::natural_domain_for_degree(pcs, 1 << log_height); + (name.to_owned(), domain, Dimensions { width: *width, height: 1 << log_height }) + }) + .collect(); + + // Get the chip ordering. + let preprocessed_chip_ordering = preprocessed_names_and_dimensions + .iter() + .enumerate() + .map(|(i, (name, _, _))| (name.to_owned(), i)) + .collect::>(); + + let vk = StarkVerifyingKey { + commit: dummy_hash(), + pc_start: BabyBear::zero(), + chip_information: preprocessed_chip_information, + chip_ordering: preprocessed_chip_ordering, + }; + + let shard_proof = ShardProof { commitment, opened_values, opening_proof, chip_ordering, chip_scopes, public_values, - } + }; + + (vk, shard_proof) } fn dummy_opened_values, A: MachineAir>( @@ -464,6 +504,7 @@ impl, SC: BabyBearFriConfigVariable> ShardProof #[cfg(any(test, feature = "export-tests"))] pub mod tests { use std::collections::VecDeque; + use std::fmt::Debug; use crate::{ challenger::{CanCopyChallenger, CanObserveVariable, DuplexChallengerVariable}, @@ -498,7 +539,7 @@ pub mod tests { type SC = BabyBearPoseidon2; pub fn build_verify_shard_with_provers< - C: CircuitConfig>, + C: CircuitConfig> + Debug, CoreP: MachineProver, RecP: MachineProver>, >( @@ -526,7 +567,6 @@ pub mod tests { let mut challenger = config.challenger_variable(&mut builder); // let vk = VerifyingKeyVariable::from_constant_key_babybear(&mut builder, &vk); Witnessable::::write(&vk, &mut witness_stream); - let vk_value = vk.clone(); let vk: VerifyingKeyVariable<_, _> = vk.read(&mut builder); vk.observe_into(&mut builder, &mut challenger); @@ -535,7 +575,7 @@ pub mod tests { .into_iter() .map(|proof| { let shape = proof.shape(); - let dummy_proof = dummy_shard_proof(&machine, &vk_value, &shape); + let (_, dummy_proof) = dummy_vk_and_shard_proof(&machine, &shape); Witnessable::::write(&proof, &mut witness_stream); dummy_proof.read(&mut builder) }) diff --git a/crates/stark/src/machine.rs b/crates/stark/src/machine.rs index d0d85e0e4e..cfe275f183 100644 --- a/crates/stark/src/machine.rs +++ b/crates/stark/src/machine.rs @@ -203,10 +203,9 @@ impl>> StarkMachine { // Order the chips and traces by trace size (biggest first), and get the ordering map. named_preprocessed_traces - .sort_by_key(|(name, trace)| Reverse((trace.height(), name.clone()))); + .sort_by_key(|(name, trace)| (Reverse(trace.height()), name.clone())); let pcs = self.config.pcs(); - let (chip_information, domains_and_traces): (Vec<_>, Vec<_>) = named_preprocessed_traces .iter() .map(|(name, trace)| { diff --git a/crates/stark/src/prover.rs b/crates/stark/src/prover.rs index 89851370f8..8a12f59c9d 100644 --- a/crates/stark/src/prover.rs +++ b/crates/stark/src/prover.rs @@ -214,7 +214,9 @@ pub trait MachineProver>: (Some(global), Some(local)) => { let (global_prover_data_idx, (global_trace, global_chip)) = global; let (local_prover_data_idx, (local_trace, local_chip)) = local; - if global_trace.height() >= local_trace.height() { + if (Reverse(global_trace.height()), global_chip) + < (Reverse(local_trace.height()), local_chip) + { merged_chips.push(global_chip.clone()); chip_scopes.push(InteractionScope::Global); merged_prover_data.push(MergedProverDataItem { @@ -327,7 +329,7 @@ where mut named_traces: Vec<(String, RowMajorMatrix>)>, ) -> ShardMainData { // Order the chips and traces by trace size (biggest first), and get the ordering map. - named_traces.sort_by_key(|(name, trace)| Reverse((trace.height(), name.clone()))); + named_traces.sort_by_key(|(name, trace)| (Reverse(trace.height()), name.clone())); let pcs = self.config().pcs(); diff --git a/crates/stark/src/types.rs b/crates/stark/src/types.rs index 6b2506bbd1..93a70111b3 100644 --- a/crates/stark/src/types.rs +++ b/crates/stark/src/types.rs @@ -204,12 +204,12 @@ impl FromIterator<(String, usize)> for ProofShape { fn from_iter>(iter: T) -> Self { let set = iter .into_iter() - .map(|(name, log_degree)| Reverse((log_degree, name))) + .map(|(name, log_degree)| (Reverse(log_degree), name)) .collect::>(); Self { chip_information: set .into_iter() - .map(|Reverse((log_degree, name))| (name, log_degree)) + .map(|(Reverse(log_degree), name)| (name, log_degree)) .collect(), } }