From 865d65ac3a9cc7e9fd7fc4255043e74a9691524b Mon Sep 17 00:00:00 2001 From: Mengran Lan Date: Tue, 27 Aug 2024 19:24:58 +0800 Subject: [PATCH 1/2] feat(prover): use params map to initialize provers (#1507) Co-authored-by: amoylan2 --- common/version/version.go | 2 +- prover/Cargo.lock | 40 +++++++++---------- prover/Cargo.toml | 4 +- prover/src/main.rs | 1 + prover/src/zk_circuits_handler.rs | 1 + prover/src/zk_circuits_handler/common.rs | 19 +++++++++ prover/src/zk_circuits_handler/darwin.rs | 43 +++++++++++++++++---- prover/src/zk_circuits_handler/darwin_v2.rs | 43 +++++++++++++++++---- 8 files changed, 116 insertions(+), 37 deletions(-) create mode 100644 prover/src/zk_circuits_handler/common.rs diff --git a/common/version/version.go b/common/version/version.go index 53c479ec7..90b47dcbf 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.4.51" +var tag = "v4.4.52" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 10dfca9ae..cff651cc3 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "ark-std 0.3.0", "bitstream-io", @@ -65,7 +65,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "ark-std 0.3.0", "bitstream-io", @@ -635,7 +635,7 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bus-mapping" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -660,7 +660,7 @@ dependencies = [ [[package]] name = "bus-mapping" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "base64 0.13.1", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "base64 0.13.1", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -1558,7 +1558,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "geth-utils 0.12.0", @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "geth-utils 0.13.0", @@ -1788,7 +1788,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "halo2_proofs", @@ -1800,7 +1800,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "halo2_proofs", @@ -1823,7 +1823,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "env_logger 0.10.2", "gobuild", @@ -1833,7 +1833,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "env_logger 0.10.2", "gobuild", @@ -2673,7 +2673,7 @@ dependencies = [ [[package]] name = "mock" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -2688,7 +2688,7 @@ dependencies = [ [[package]] name = "mock" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -2703,7 +2703,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "halo2curves", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "halo2curves", @@ -3309,7 +3309,7 @@ dependencies = [ [[package]] name = "prover" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "aggregator 0.12.0", "anyhow", @@ -3343,7 +3343,7 @@ dependencies = [ [[package]] name = "prover" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "aggregator 0.13.0", "anyhow", @@ -5338,7 +5338,7 @@ dependencies = [ [[package]] name = "zkevm-circuits" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "array-init", "bus-mapping 0.12.0", @@ -5380,7 +5380,7 @@ dependencies = [ [[package]] name = "zkevm-circuits" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "array-init", "bus-mapping 0.13.0", diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 96b60de9b..36db6a39b 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -29,8 +29,8 @@ ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = " ethers-providers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" } halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1" } snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop", default-features = false, features = ["loader_halo2", "loader_evm", "halo2-pse"] } -prover_darwin = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.12.0", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } -prover_darwin_v2 = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.13.0", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } +prover_darwin = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.12.2", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } +prover_darwin_v2 = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.13.1", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } base64 = "0.13.1" reqwest = { version = "0.12.4", features = ["gzip"] } reqwest-middleware = "0.3" diff --git a/prover/src/main.rs b/prover/src/main.rs index d1bf82830..75553187a 100644 --- a/prover/src/main.rs +++ b/prover/src/main.rs @@ -1,4 +1,5 @@ #![feature(lazy_cell)] +#![feature(core_intrinsics)] mod config; mod coordinator_client; diff --git a/prover/src/zk_circuits_handler.rs b/prover/src/zk_circuits_handler.rs index 06622d9a7..d1a8eb38c 100644 --- a/prover/src/zk_circuits_handler.rs +++ b/prover/src/zk_circuits_handler.rs @@ -1,3 +1,4 @@ +mod common; mod darwin; mod darwin_v2; diff --git a/prover/src/zk_circuits_handler/common.rs b/prover/src/zk_circuits_handler/common.rs new file mode 100644 index 000000000..8bd192e8c --- /dev/null +++ b/prover/src/zk_circuits_handler/common.rs @@ -0,0 +1,19 @@ +use std::{collections::BTreeMap, rc::Rc}; + +use once_cell::sync::OnceCell; + +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; + +static mut PARAMS_MAP: OnceCell>>> = OnceCell::new(); + +pub fn get_params_map<'a, F>(load_params_func: F) -> &'a BTreeMap> +where + F: FnOnce() -> BTreeMap>, +{ + unsafe { + PARAMS_MAP.get_or_init(|| { + let params_map = load_params_func(); + Rc::new(params_map) + }) + } +} diff --git a/prover/src/zk_circuits_handler/darwin.rs b/prover/src/zk_circuits_handler/darwin.rs index e18dc2474..da5b5f88d 100644 --- a/prover/src/zk_circuits_handler/darwin.rs +++ b/prover/src/zk_circuits_handler/darwin.rs @@ -11,9 +11,9 @@ use crate::types::{CommonHash, Task}; use std::{cell::RefCell, cmp::Ordering, env, rc::Rc}; use prover_darwin::{ - aggregator::Prover as BatchProver, check_chunk_hashes, zkevm::Prover as ChunkProver, - BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo, - ChunkProof, ChunkProvingTask, + aggregator::Prover as BatchProver, check_chunk_hashes, common::Prover as CommonProver, + zkevm::Prover as ChunkProver, BatchProof, BatchProvingTask, BlockTrace, BundleProof, + BundleProvingTask, ChunkInfo, ChunkProof, ChunkProvingTask, }; // Only used for debugging. @@ -39,8 +39,8 @@ fn get_block_number(block_trace: &BlockTrace) -> Option { #[derive(Default)] pub struct DarwinHandler { - chunk_prover: Option>, - batch_prover: Option>, + chunk_prover: Option>>, + batch_prover: Option>>, geth_client: Option>>, } @@ -52,15 +52,44 @@ impl DarwinHandler { assets_dir: &str, geth_client: Option>>, ) -> Result { + let class_name = std::intrinsics::type_name::(); match prover_type { ProverType::Chunk => Ok(Self { - chunk_prover: Some(RefCell::new(ChunkProver::from_dirs(params_dir, assets_dir))), + chunk_prover: { + let degrees = prover_darwin::config::ZKEVM_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(ChunkProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, batch_prover: None, geth_client, }), ProverType::Batch => Ok(Self { - batch_prover: Some(RefCell::new(BatchProver::from_dirs(params_dir, assets_dir))), + batch_prover: { + let degrees = prover_darwin::config::AGG_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(BatchProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, chunk_prover: None, geth_client, }), diff --git a/prover/src/zk_circuits_handler/darwin_v2.rs b/prover/src/zk_circuits_handler/darwin_v2.rs index 9c7ce6910..616d57b20 100644 --- a/prover/src/zk_circuits_handler/darwin_v2.rs +++ b/prover/src/zk_circuits_handler/darwin_v2.rs @@ -11,9 +11,9 @@ use crate::types::{CommonHash, Task}; use std::{cell::RefCell, cmp::Ordering, env, rc::Rc}; use prover_darwin_v2::{ - aggregator::Prover as BatchProver, check_chunk_hashes, zkevm::Prover as ChunkProver, - BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo, - ChunkProof, ChunkProvingTask, + aggregator::Prover as BatchProver, check_chunk_hashes, common::Prover as CommonProver, + zkevm::Prover as ChunkProver, BatchProof, BatchProvingTask, BlockTrace, BundleProof, + BundleProvingTask, ChunkInfo, ChunkProof, ChunkProvingTask, }; // Only used for debugging. @@ -39,8 +39,8 @@ fn get_block_number(block_trace: &BlockTrace) -> Option { #[derive(Default)] pub struct DarwinV2Handler { - chunk_prover: Option>, - batch_prover: Option>, + chunk_prover: Option>>, + batch_prover: Option>>, geth_client: Option>>, } @@ -52,15 +52,44 @@ impl DarwinV2Handler { assets_dir: &str, geth_client: Option>>, ) -> Result { + let class_name = std::intrinsics::type_name::(); match prover_type { ProverType::Chunk => Ok(Self { - chunk_prover: Some(RefCell::new(ChunkProver::from_dirs(params_dir, assets_dir))), + chunk_prover: { + let degrees = prover_darwin_v2::config::ZKEVM_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(ChunkProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, batch_prover: None, geth_client, }), ProverType::Batch => Ok(Self { - batch_prover: Some(RefCell::new(BatchProver::from_dirs(params_dir, assets_dir))), + batch_prover: { + let degrees = prover_darwin_v2::config::AGG_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(BatchProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, chunk_prover: None, geth_client, }), From 432c98b6514682ba2f6ef7466286ef385d00b6ed Mon Sep 17 00:00:00 2001 From: sbaizet <74511063+sbaizet-ledger@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:12:04 +0200 Subject: [PATCH 2/2] add ca-certificates package to bridge-history services (#1506) --- build/dockerfiles/bridgehistoryapi-db-cli.Dockerfile | 2 ++ build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build/dockerfiles/bridgehistoryapi-db-cli.Dockerfile b/build/dockerfiles/bridgehistoryapi-db-cli.Dockerfile index 6797d9e92..cf728837f 100644 --- a/build/dockerfiles/bridgehistoryapi-db-cli.Dockerfile +++ b/build/dockerfiles/bridgehistoryapi-db-cli.Dockerfile @@ -15,5 +15,7 @@ RUN --mount=target=. \ # Pull db_cli into a second stage deploy alpine container FROM alpine:latest COPY --from=builder /bin/db_cli /bin/ +RUN apk update && apk add ca-certificates +RUN update-ca-certificates WORKDIR /app ENTRYPOINT ["db_cli"] \ No newline at end of file diff --git a/build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile b/build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile index 68815e63c..991b2b11a 100644 --- a/build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile +++ b/build/dockerfiles/bridgehistoryapi-fetcher.Dockerfile @@ -17,7 +17,8 @@ RUN --mount=target=. \ FROM ubuntu:20.04 ENV CGO_LDFLAGS="-Wl,--no-as-needed -ldl" - +RUN apk update && apk add ca-certificates +RUN update-ca-certificates COPY --from=builder /bin/bridgehistoryapi-fetcher /bin/ WORKDIR /app ENTRYPOINT ["bridgehistoryapi-fetcher"]