Skip to content

Commit

Permalink
preliminary (WIP)
Browse files Browse the repository at this point in the history
  • Loading branch information
@roynalnaruto
roynalnaruto committed Oct 23, 2024
1 parent bc8f9db commit 27eaebd
Show file tree
Hide file tree
Showing 12 changed files with 821 additions and 697 deletions.
527 changes: 392 additions & 135 deletions prover/Cargo.lock
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions prover/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "
ethers-providers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1" }
snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop", default-features = false, features = ["loader_halo2", "loader_evm", "halo2-pse"] }
prover_darwin = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.12.2", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] }
prover_darwin_v2 = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.13.1", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] }
prover_darwin = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.13.1", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] }
prover_euclid = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "feat/hybrid-snark-agg", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] }
base64 = "0.13.1"
reqwest = { version = "0.12.4", features = ["gzip"] }
reqwest-middleware = "0.3"
Expand Down
1 change: 0 additions & 1 deletion prover/rust-toolchain
View file

This file was deleted.

2 changes: 2 additions & 0 deletions prover/rust-toolchain.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[toolchain]
channel = "nightly-2024-01-25"
1 change: 1 addition & 0 deletions prover/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#![feature(lazy_cell)]
#![allow(internal_features)]
#![feature(core_intrinsics)]

mod config;
Expand Down
10 changes: 8 additions & 2 deletions prover/src/prover.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@ impl<'a> Prover<'a> {
let keystore_path = &config.keystore_path;
let keystore_password = &config.keystore_password;

let geth_client = if config.prover_type == ProverType::Chunk {
let geth_client = if matches!(
config.prover_type,
ProverType::ChunkHalo2 | ProverType::ChunkSp1 | ProverType::ChunkAll
) {
Some(Rc::new(RefCell::new(
GethClient::new(
&config.prover_name,
Expand Down Expand Up @@ -73,7 +76,10 @@ impl<'a> Prover<'a> {
prover_height: None,
};

if self.config.prover_type == ProverType::Chunk {
if matches!(
self.config.prover_type,
ProverType::ChunkHalo2 | ProverType::ChunkSp1 | ProverType::ChunkAll
) {
let latest_block_number = self.get_latest_block_number_value()?;
if let Some(v) = latest_block_number {
if v.as_u64() == 0 {
Expand Down
25 changes: 18 additions & 7 deletions prover/src/types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,19 @@ pub type CommonHash = H256;
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum TaskType {
Undefined,
Chunk,
ChunkHalo2,
ChunkSp1,
Batch,
Bundle,
}

impl TaskType {
fn from_u8(v: u8) -> Self {
match v {
1 => TaskType::Chunk,
1 => TaskType::ChunkHalo2,
2 => TaskType::Batch,
3 => TaskType::Bundle,
4 => TaskType::ChunkSp1,
_ => TaskType::Undefined,
}
}
Expand All @@ -31,9 +33,10 @@ impl Serialize for TaskType {
{
match *self {
TaskType::Undefined => serializer.serialize_u8(0),
TaskType::Chunk => serializer.serialize_u8(1),
TaskType::ChunkHalo2 => serializer.serialize_u8(1),
TaskType::Batch => serializer.serialize_u8(2),
TaskType::Bundle => serializer.serialize_u8(3),
TaskType::ChunkSp1 => serializer.serialize_u8(4),
}
}
}
Expand All @@ -56,15 +59,19 @@ impl Default for TaskType {

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ProverType {
Chunk,
ChunkHalo2,
ChunkSp1,
ChunkAll,
Batch,
}

impl ProverType {
fn from_u8(v: u8) -> Self {
match v {
1 => ProverType::Chunk,
1 => ProverType::ChunkHalo2,
2 => ProverType::Batch,
3 => ProverType::ChunkSp1,
4 => ProverType::ChunkAll,
_ => {
panic!("invalid prover_type")
}
Expand All @@ -73,8 +80,10 @@ impl ProverType {

pub fn to_u8(self) -> u8 {
match self {
ProverType::Chunk => 1,
ProverType::ChunkHalo2 => 1,
ProverType::Batch => 2,
ProverType::ChunkSp1 => 3,
ProverType::ChunkAll => 4,
}
}
}
Expand All @@ -85,8 +94,10 @@ impl Serialize for ProverType {
S: Serializer,
{
match *self {
ProverType::Chunk => serializer.serialize_u8(1),
ProverType::ChunkHalo2 => serializer.serialize_u8(1),
ProverType::Batch => serializer.serialize_u8(2),
ProverType::ChunkSp1 => serializer.serialize_u8(3),
ProverType::ChunkAll => serializer.serialize_u8(4),
}
}
}
Expand Down
4 changes: 3 additions & 1 deletion prover/src/utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ pub fn log_init(log_file: Option<String>) {

pub fn get_task_types(prover_type: ProverType) -> Vec<TaskType> {
match prover_type {
ProverType::Chunk => vec![TaskType::Chunk],
ProverType::ChunkHalo2 => vec![TaskType::ChunkHalo2],
ProverType::ChunkSp1 => vec![TaskType::ChunkSp1],
ProverType::ChunkAll => vec![TaskType::ChunkHalo2, TaskType::ChunkSp1],
ProverType::Batch => vec![TaskType::Batch, TaskType::Bundle],
}
}
6 changes: 3 additions & 3 deletions prover/src/zk_circuits_handler.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
mod common;
mod darwin;
mod darwin_v2;
mod euclid;

use super::geth_client::GethClient;
use crate::{
Expand All @@ -10,7 +10,7 @@ use crate::{
};
use anyhow::{bail, Result};
use darwin::DarwinHandler;
use darwin_v2::DarwinV2Handler;
use euclid::EuclidHandler;
use std::{cell::RefCell, collections::HashMap, rc::Rc};

type HardForkName = String;
Expand Down Expand Up @@ -84,7 +84,7 @@ impl<'a> CircuitsHandlerProvider<'a> {
&config.high_version_circuit.hard_fork_name
);
AssetsDirEnvConfig::enable_second();
DarwinV2Handler::new(
EuclidHandler::new(
prover_type,
&config.high_version_circuit.params_path,
&config.high_version_circuit.assets_path,
Expand Down
102 changes: 81 additions & 21 deletions prover/src/zk_circuits_handler/darwin.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use prover_darwin::{
static OUTPUT_DIR: Lazy<Option<String>> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok());

#[derive(Debug, Clone, Deserialize)]
pub struct BatchTaskDetail {
struct BatchTaskDetail {
pub chunk_infos: Vec<ChunkInfo>,
#[serde(flatten)]
pub batch_proving_task: BatchProvingTask,
Expand Down Expand Up @@ -66,8 +66,9 @@ impl DarwinHandler {
geth_client,
};
let degrees: Vec<u32> = get_degrees(&prover_types_set, |prover_type| match prover_type {
ProverType::Chunk => ZKEVM_DEGREES.clone(),
ProverType::ChunkHalo2 => ZKEVM_DEGREES.clone(),
ProverType::Batch => AGG_DEGREES.clone(),
_ => unreachable!("Darwin supports prover types in [ChunkHalo2 and Batch] only"),
});
let params_map = get_params_map_instance(|| {
log::info!(
Expand All @@ -80,7 +81,7 @@ impl DarwinHandler {
});
for prover_type in prover_types_set {
match prover_type {
ProverType::Chunk => {
ProverType::ChunkHalo2 => {
handler.chunk_prover = Some(RefCell::new(ChunkProver::from_params_and_assets(
params_map, assets_dir,
)));
Expand All @@ -90,6 +91,7 @@ impl DarwinHandler {
params_map, assets_dir,
)))
}
_ => unreachable!("Darwin supports prover types in [ChunkHalo2 and Batch] only"),
}
}
Ok(handler)
Expand Down Expand Up @@ -247,7 +249,7 @@ impl DarwinHandler {
impl CircuitsHandler for DarwinHandler {
fn get_vk(&self, task_type: TaskType) -> Option<Vec<u8>> {
match task_type {
TaskType::Chunk => self
TaskType::ChunkHalo2 => self
.chunk_prover
.as_ref()
.and_then(|prover| prover.borrow().get_vk()),
Expand All @@ -265,7 +267,7 @@ impl CircuitsHandler for DarwinHandler {

fn get_proof_data(&self, task_type: TaskType, task: &crate::types::Task) -> Result<String> {
match task_type {
TaskType::Chunk => self.gen_chunk_proof(task),
TaskType::ChunkHalo2 => self.gen_chunk_proof(task),
TaskType::Batch => self.gen_batch_proof(task),
TaskType::Bundle => self.gen_bundle_proof(task),
_ => unreachable!(),
Expand All @@ -279,7 +281,11 @@ impl CircuitsHandler for DarwinHandler {
mod tests {
use super::*;
use crate::zk_circuits_handler::utils::encode_vk;
use prover_darwin::utils::chunk_trace_to_witness_block;
use ethers_core::types::H256;
use prover_darwin_v2::{
aggregator::eip4844, utils::chunk_trace_to_witness_block, BatchData, BatchHeader,
MAX_AGG_SNARKS,
};
use std::{path::PathBuf, sync::LazyLock};

#[ctor::ctor]
Expand All @@ -290,7 +296,7 @@ mod tests {

static DEFAULT_WORK_DIR: &str = "/assets";
static WORK_DIR: LazyLock<String> = LazyLock::new(|| {
std::env::var("DARWIN_TEST_DIR")
std::env::var("DARWIN_V2_TEST_DIR")
.unwrap_or(String::from(DEFAULT_WORK_DIR))
.trim_end_matches('/')
.to_string()
Expand All @@ -315,7 +321,7 @@ mod tests {
#[test]
fn test_circuits() -> Result<()> {
let bi_handler = DarwinHandler::new_multi(
vec![ProverType::Chunk, ProverType::Batch],
vec![ProverType::ChunkHalo2, ProverType::Batch],
&PARAMS_PATH,
&ASSETS_PATH,
None,
Expand All @@ -327,13 +333,14 @@ mod tests {
check_vk(TaskType::Chunk, chunk_vk, "chunk vk must be available");
let chunk_dir_paths = get_chunk_dir_paths()?;
log::info!("chunk_dir_paths, {:?}", chunk_dir_paths);
let mut chunk_traces = vec![];
let mut chunk_infos = vec![];
let mut chunk_proofs = vec![];
for (id, chunk_path) in chunk_dir_paths.into_iter().enumerate() {
let chunk_id = format!("chunk_proof{}", id + 1);
log::info!("start to process {chunk_id}");
let chunk_trace = read_chunk_trace(chunk_path)?;

chunk_traces.push(chunk_trace.clone());
let chunk_info = traces_to_chunk_info(chunk_trace.clone())?;
chunk_infos.push(chunk_info);

Expand All @@ -347,7 +354,7 @@ mod tests {
let batch_handler = chunk_handler;
let batch_vk = batch_handler.get_vk(TaskType::Batch).unwrap();
check_vk(TaskType::Batch, batch_vk, "batch vk must be available");
let batch_task_detail = make_batch_task_detail(chunk_infos, chunk_proofs);
let batch_task_detail = make_batch_task_detail(chunk_traces, chunk_proofs, None);
log::info!("start to prove batch");
let batch_proof = batch_handler.gen_batch_proof_raw(batch_task_detail)?;
let proof_data = serde_json::to_string(&batch_proof)?;
Expand All @@ -356,17 +363,70 @@ mod tests {
Ok(())
}

fn make_batch_task_detail(_: Vec<ChunkInfo>, _: Vec<ChunkProof>) -> BatchTaskDetail {
todo!();
// BatchTaskDetail {
// chunk_infos,
// batch_proving_task: BatchProvingTask {
// parent_batch_hash: todo!(),
// parent_state_root: todo!(),
// batch_header: todo!(),
// chunk_proofs,
// },
// }
// copied from https://github.com/scroll-tech/scroll-prover/blob/main/integration/src/prove.rs
fn get_blob_from_chunks(chunks: &[ChunkInfo]) -> Vec<u8> {
let num_chunks = chunks.len();

let padded_chunk =
ChunkInfo::mock_padded_chunk_info_for_testing(chunks.last().as_ref().unwrap());
let chunks_with_padding = [
chunks.to_vec(),
vec![padded_chunk; MAX_AGG_SNARKS - num_chunks],
]
.concat();
let batch_data = BatchData::<{ MAX_AGG_SNARKS }>::new(chunks.len(), &chunks_with_padding);
let batch_bytes = batch_data.get_batch_data_bytes();
let blob_bytes = eip4844::get_blob_bytes(&batch_bytes);
log::info!("blob_bytes len {}", blob_bytes.len());
blob_bytes
}

// TODO: chunk_infos can be extracted from chunk_proofs.
// Still needed?
fn make_batch_task_detail(
chunk_traces: Vec<Vec<BlockTrace>>,
chunk_proofs: Vec<ChunkProof>,
last_batcher_header: Option<BatchHeader<{ MAX_AGG_SNARKS }>>,
) -> BatchTaskDetail {
// dummy parent batch hash
let dummy_parent_batch_hash = H256([
0xab, 0xac, 0xad, 0xae, 0xaf, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
]);
let chunk_infos: Vec<_> = chunk_proofs.iter().map(|p| p.chunk_info.clone()).collect();

let l1_message_popped = chunk_traces
.iter()
.flatten()
.map(|chunk| chunk.num_l1_txs())
.sum();
let last_block_timestamp = chunk_traces.last().map_or(0, |block_traces| {
block_traces
.last()
.map_or(0, |block_trace| block_trace.header.timestamp.as_u64())
});

let blob_bytes = get_blob_from_chunks(&chunk_infos);
let batch_header = BatchHeader::construct_from_chunks(
last_batcher_header.map_or(4, |header| header.version),
last_batcher_header.map_or(123, |header| header.batch_index + 1),
l1_message_popped,
last_batcher_header.map_or(l1_message_popped, |header| {
header.total_l1_message_popped + l1_message_popped
}),
last_batcher_header.map_or(dummy_parent_batch_hash, |header| header.batch_hash()),
last_block_timestamp,
&chunk_infos,
&blob_bytes,
);
BatchTaskDetail {
chunk_infos,
batch_proving_task: BatchProvingTask {
chunk_proofs,
batch_header,
blob_bytes,
},
}
}

fn check_vk(proof_type: TaskType, vk: Vec<u8>, info: &str) {
Expand Down
Loading

0 comments on commit 27eaebd

Please sign in to comment.