From 922d679a12974c0ded352c02b5f1234913a87b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erce=20Can=20Bekt=C3=BCre?= <47954181+ercecan@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:07:01 +0300 Subject: [PATCH 1/2] Remove accept public input as proven field (#39) --- src/citrea_config/rollup.rs | 2 -- src/framework.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/src/citrea_config/rollup.rs b/src/citrea_config/rollup.rs index 3342686..9213778 100644 --- a/src/citrea_config/rollup.rs +++ b/src/citrea_config/rollup.rs @@ -12,8 +12,6 @@ pub struct RunnerConfig { pub sequencer_client_url: String, /// Saves sequencer soft confirmations if set to true pub include_tx_body: bool, - /// Only true for tests - pub accept_public_input_as_proven: Option, /// Number of blocks to request during sync #[serde(default = "default_sync_blocks_count")] pub sync_blocks_count: u64, diff --git a/src/framework.rs b/src/framework.rs index d782157..39cf753 100644 --- a/src/framework.rs +++ b/src/framework.rs @@ -353,7 +353,6 @@ impl TestFramework { runner_bind_host, sequencer_rollup.rpc.bind_port, ), include_tx_body: true, - accept_public_input_as_proven: Some(true), sync_blocks_count: 10, pruning_config: None, }); From c605312662d3fc8dd60bb4cd195b97676793ae95 Mon Sep 17 00:00:00 2001 From: jfldde <168934971+jfldde@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:54:21 +0000 Subject: [PATCH 2/2] Reorg support (#42) --- src/bitcoin.rs | 29 ++- src/config/docker.rs | 11 +- src/docker.rs | 3 +- src/framework.rs | 420 +++++++++++++++++++++---------------------- src/node.rs | 2 +- src/test_case.rs | 32 ++-- src/traits.rs | 3 +- src/utils.rs | 3 +- tests/mod.rs | 12 +- 9 files changed, 270 insertions(+), 245 deletions(-) diff --git a/src/bitcoin.rs b/src/bitcoin.rs index 51559e5..2a2fb57 100644 --- a/src/bitcoin.rs +++ b/src/bitcoin.rs @@ -306,8 +306,9 @@ impl BitcoinNodeCluster { Ok(()) } - pub async fn wait_for_sync(&self, timeout: Duration) -> Result<()> { + pub async fn wait_for_sync(&self, timeout: Option) -> Result<()> { let start = Instant::now(); + let timeout = timeout.unwrap_or(Duration::from_secs(30)); while start.elapsed() < timeout { let mut heights = HashSet::new(); for node in &self.inner { @@ -326,16 +327,34 @@ impl BitcoinNodeCluster { // Connect all bitcoin nodes between them pub async fn connect_nodes(&self) -> Result<()> { - for (i, from_node) in self.inner.iter().enumerate() { - for (j, to_node) in self.inner.iter().enumerate() { + for (i, from_node) in self.iter().enumerate() { + for (j, to_node) in self.iter().enumerate() { + if i != j { + let ip = match &to_node.spawn_output { + SpawnOutput::Container(container) => container.ip.clone(), + SpawnOutput::Child(_) => "127.0.0.1".to_string(), + }; + + let target_node_addr = format!("{}:{}", ip, to_node.config.p2p_port); + from_node.onetry_node(&target_node_addr).await?; + } + } + } + Ok(()) + } + + pub async fn disconnect_nodes(&self) -> Result<()> { + for (i, from_node) in self.iter().enumerate() { + for (j, to_node) in self.iter().enumerate() { if i != j { let ip = match &to_node.spawn_output { SpawnOutput::Container(container) => container.ip.clone(), SpawnOutput::Child(_) => "127.0.0.1".to_string(), }; - let add_node_arg = format!("{}:{}", ip, to_node.config.p2p_port); - from_node.add_node(&add_node_arg).await?; + let target_node_addr = format!("{}:{}", ip, to_node.config.p2p_port); + // from_node.remove_node(&target_node_addr).await?; + from_node.disconnect_node(&target_node_addr).await?; } } } diff --git a/src/config/docker.rs b/src/config/docker.rs index 87fa33d..b5459e1 100644 --- a/src/config/docker.rs +++ b/src/config/docker.rs @@ -1,13 +1,14 @@ -use std::fmt::Debug; -use std::path::PathBuf; +use std::{fmt::Debug, path::PathBuf}; use serde::Serialize; use tracing::debug; use super::{BitcoinConfig, FullL2NodeConfig, NodeKindMarker}; -use crate::log_provider::LogPathProvider; -use crate::node::{get_citrea_args, Config, NodeKind}; -use crate::utils::get_genesis_path; +use crate::{ + log_provider::LogPathProvider, + node::{get_citrea_args, Config, NodeKind}, + utils::get_genesis_path, +}; const DEFAULT_BITCOIN_DOCKER_IMAGE: &str = "bitcoin/bitcoin:28.0"; const DEFAULT_CITREA_DOCKER_IMAGE: &str = diff --git a/src/docker.rs b/src/docker.rs index 06a991f..ebef961 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -18,9 +18,8 @@ use futures::StreamExt; use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex, task::JoinHandle}; use tracing::{debug, error, info}; -use crate::{config::TestCaseDockerConfig, node::NodeKind}; - use super::{config::DockerConfig, traits::SpawnOutput, utils::generate_test_id}; +use crate::{config::TestCaseDockerConfig, node::NodeKind}; #[derive(Debug)] pub struct ContainerSpawnOutput { diff --git a/src/framework.rs b/src/framework.rs index 39cf753..8eec3c3 100644 --- a/src/framework.rs +++ b/src/framework.rs @@ -10,28 +10,28 @@ use tracing::{debug, info}; use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; use super::{ - bitcoin::BitcoinNodeCluster, docker::DockerEnv, full_node::FullNode, node::NodeKind, - sequencer::Sequencer, traits::NodeT, Result, -}; -use super::{ + bitcoin::BitcoinNodeCluster, config::{ BitcoinConfig, FullBatchProverConfig, FullFullNodeConfig, FullSequencerConfig, RollupConfig, TestCaseConfig, TestConfig, }, + docker::DockerEnv, + full_node::FullNode, + node::NodeKind, + sequencer::Sequencer, + traits::NodeT, utils::{copy_directory, get_available_port}, + Result, }; use crate::{ batch_prover::BatchProver, - light_client_prover::LightClientProver, - log_provider::{LogPathProvider, LogPathProviderErased}, - test_case::TestCase, - utils::tail_file, -}; -use crate::{ config::{ BitcoinServiceConfig, FullLightClientProverConfig, RpcConfig, RunnerConfig, StorageConfig, }, - utils::{get_default_genesis_path, get_workspace_root}, + light_client_prover::LightClientProver, + log_provider::{LogPathProvider, LogPathProviderErased}, + test_case::TestCase, + utils::{get_default_genesis_path, get_workspace_root, tail_file}, }; pub struct TestContext { @@ -76,7 +76,7 @@ impl TestFramework { } else { None }; - let config = Self::generate_test_config::(test_case, &docker)?; + let config = generate_test_config::(test_case, &docker)?; anyhow::ensure!( config.test_case.n_nodes > 0, @@ -265,209 +265,207 @@ impl TestFramework { self.initial_da_height = da.get_block_count().await?; Ok(()) } +} - fn generate_test_config( - test_case: TestCaseConfig, - docker: &Option, - ) -> Result { - let env = T::test_env(); - let bitcoin = T::bitcoin_config(); - let batch_prover = T::batch_prover_config(); - let light_client_prover = T::light_client_prover_config(); - let sequencer = T::sequencer_config(); - let sequencer_rollup = RollupConfig::default(); - let batch_prover_rollup = RollupConfig::default(); - let light_client_prover_rollup = RollupConfig::default(); - let full_node_rollup = RollupConfig::default(); - - let [bitcoin_dir, dbs_dir, batch_prover_dir, light_client_prover_dir, sequencer_dir, full_node_dir, genesis_dir, tx_backup_dir] = - create_dirs(&test_case.dir)?; - - copy_genesis_dir(&test_case.genesis_dir, &genesis_dir)?; - - let mut bitcoin_confs = vec![]; - for i in 0..test_case.n_nodes { - let data_dir = bitcoin_dir.join(i.to_string()); - std::fs::create_dir_all(&data_dir) - .with_context(|| format!("Failed to create {} directory", data_dir.display()))?; - - let p2p_port = get_available_port()?; - let rpc_port = get_available_port()?; - - bitcoin_confs.push(BitcoinConfig { - p2p_port, - rpc_port, - data_dir, - env: env.bitcoin().clone(), - idx: i, - ..bitcoin.clone() - }); - } - - bitcoin_confs[0].docker_host = docker - .as_ref() - .and_then(|d| d.citrea().then(|| d.get_hostname(&NodeKind::Bitcoin))); - - // Target first bitcoin node as DA for now - let da_config: BitcoinServiceConfig = bitcoin_confs[0].clone().into(); - - let runner_bind_host = match docker.as_ref() { - Some(d) if d.citrea() => d.get_hostname(&NodeKind::Sequencer), - _ => sequencer_rollup.rpc.bind_host.clone(), - }; - - let bind_host = match docker.as_ref() { - Some(d) if d.citrea() => "0.0.0.0".to_string(), - _ => sequencer_rollup.rpc.bind_host.clone(), - }; - - let sequencer_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::Sequencer.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - da_private_key: Some( - "045FFC81A3C1FDB3AF1359DBF2D114B0B3EFBF7F29CC9C5DA01267AA39D2C78D" - .to_string(), - ), - node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), - tx_backup_dir: tx_backup_dir.display().to_string(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{node_kind}-db")), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - bind_host: bind_host.clone(), - ..sequencer_rollup.rpc - }, - ..sequencer_rollup - } - }; - - let runner_config = Some(RunnerConfig { - sequencer_client_url: format!( - "http://{}:{}", - runner_bind_host, sequencer_rollup.rpc.bind_port, - ), - include_tx_body: true, - sync_blocks_count: 10, - pruning_config: None, +fn generate_test_config( + test_case: TestCaseConfig, + docker: &Option, +) -> Result { + let env = T::test_env(); + let bitcoin = T::bitcoin_config(); + let batch_prover = T::batch_prover_config(); + let light_client_prover = T::light_client_prover_config(); + let sequencer = T::sequencer_config(); + let sequencer_rollup = RollupConfig::default(); + let batch_prover_rollup = RollupConfig::default(); + let light_client_prover_rollup = RollupConfig::default(); + let full_node_rollup = RollupConfig::default(); + + let [bitcoin_dir, dbs_dir, batch_prover_dir, light_client_prover_dir, sequencer_dir, full_node_dir, genesis_dir, tx_backup_dir] = + create_dirs(&test_case.dir)?; + + copy_genesis_dir(&test_case.genesis_dir, &genesis_dir)?; + + let mut bitcoin_confs = vec![]; + for i in 0..test_case.n_nodes { + let data_dir = bitcoin_dir.join(i.to_string()); + std::fs::create_dir_all(&data_dir) + .with_context(|| format!("Failed to create {} directory", data_dir.display()))?; + + let p2p_port = get_available_port()?; + let rpc_port = get_available_port()?; + + bitcoin_confs.push(BitcoinConfig { + p2p_port, + rpc_port, + data_dir, + env: env.bitcoin().clone(), + idx: i, + ..bitcoin.clone() }); + } - let batch_prover_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::BatchProver.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - da_private_key: Some( - "75BAF964D074594600366E5B111A1DA8F86B2EFE2D22DA51C8D82126A0FCAC72" - .to_string(), - ), - node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), - tx_backup_dir: tx_backup_dir.display().to_string(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{node_kind}-db")), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - bind_host: bind_host.clone(), - ..batch_prover_rollup.rpc - }, - runner: runner_config.clone(), - ..batch_prover_rollup - } - }; - - let light_client_prover_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::LightClientProver.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - da_private_key: None, - node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), - tx_backup_dir: tx_backup_dir.display().to_string(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{node_kind}-db")), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - bind_host: bind_host.clone(), - ..light_client_prover_rollup.rpc - }, - runner: runner_config.clone(), - ..light_client_prover_rollup - } - }; - - let full_node_rollup = { - let bind_port = get_available_port()?; - let node_kind = NodeKind::FullNode.to_string(); - RollupConfig { - da: BitcoinServiceConfig { - node_url: format!( - "http://{}/wallet/{}", - da_config.node_url, - NodeKind::Bitcoin // Use default wallet - ), - tx_backup_dir: tx_backup_dir.display().to_string(), - ..da_config.clone() - }, - storage: StorageConfig { - path: dbs_dir.join(format!("{node_kind}-db")), - db_max_open_files: None, - }, - rpc: RpcConfig { - bind_port, - bind_host: bind_host.clone(), - ..full_node_rollup.rpc - }, - runner: runner_config.clone(), - ..full_node_rollup - } - }; + bitcoin_confs[0].docker_host = docker + .as_ref() + .and_then(|d| d.citrea().then(|| d.get_hostname(&NodeKind::Bitcoin))); + + // Target first bitcoin node as DA for now + let da_config: BitcoinServiceConfig = bitcoin_confs[0].clone().into(); + + let runner_bind_host = match docker.as_ref() { + Some(d) if d.citrea() => d.get_hostname(&NodeKind::Sequencer), + _ => sequencer_rollup.rpc.bind_host.clone(), + }; + + let bind_host = match docker.as_ref() { + Some(d) if d.citrea() => "0.0.0.0".to_string(), + _ => sequencer_rollup.rpc.bind_host.clone(), + }; + + let sequencer_rollup = { + let bind_port = get_available_port()?; + let node_kind = NodeKind::Sequencer.to_string(); + RollupConfig { + da: BitcoinServiceConfig { + da_private_key: Some( + "045FFC81A3C1FDB3AF1359DBF2D114B0B3EFBF7F29CC9C5DA01267AA39D2C78D".to_string(), + ), + node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), + tx_backup_dir: tx_backup_dir.display().to_string(), + ..da_config.clone() + }, + storage: StorageConfig { + path: dbs_dir.join(format!("{node_kind}-db")), + db_max_open_files: None, + }, + rpc: RpcConfig { + bind_port, + bind_host: bind_host.clone(), + ..sequencer_rollup.rpc + }, + ..sequencer_rollup + } + }; + + let runner_config = Some(RunnerConfig { + sequencer_client_url: format!( + "http://{}:{}", + runner_bind_host, sequencer_rollup.rpc.bind_port, + ), + include_tx_body: true, + sync_blocks_count: 10, + pruning_config: None, + }); - Ok(TestConfig { - bitcoin: bitcoin_confs, - sequencer: FullSequencerConfig::new( - sequencer, - sequencer_rollup, - None, - sequencer_dir, - env.sequencer(), - )?, - batch_prover: FullBatchProverConfig::new( - batch_prover, - batch_prover_rollup, - None, - batch_prover_dir, - env.batch_prover(), - )?, - light_client_prover: FullLightClientProverConfig::new( - light_client_prover, - light_client_prover_rollup, - None, - light_client_prover_dir, - env.light_client_prover(), - )?, - full_node: FullFullNodeConfig::new( - (), - full_node_rollup, - None, - full_node_dir, - env.full_node(), - )?, - test_case, - }) - } + let batch_prover_rollup = { + let bind_port = get_available_port()?; + let node_kind = NodeKind::BatchProver.to_string(); + RollupConfig { + da: BitcoinServiceConfig { + da_private_key: Some( + "75BAF964D074594600366E5B111A1DA8F86B2EFE2D22DA51C8D82126A0FCAC72".to_string(), + ), + node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), + tx_backup_dir: tx_backup_dir.display().to_string(), + ..da_config.clone() + }, + storage: StorageConfig { + path: dbs_dir.join(format!("{node_kind}-db")), + db_max_open_files: None, + }, + rpc: RpcConfig { + bind_port, + bind_host: bind_host.clone(), + ..batch_prover_rollup.rpc + }, + runner: runner_config.clone(), + ..batch_prover_rollup + } + }; + + let light_client_prover_rollup = { + let bind_port = get_available_port()?; + let node_kind = NodeKind::LightClientProver.to_string(); + RollupConfig { + da: BitcoinServiceConfig { + da_private_key: None, + node_url: format!("http://{}/wallet/{}", da_config.node_url, node_kind), + tx_backup_dir: tx_backup_dir.display().to_string(), + ..da_config.clone() + }, + storage: StorageConfig { + path: dbs_dir.join(format!("{node_kind}-db")), + db_max_open_files: None, + }, + rpc: RpcConfig { + bind_port, + bind_host: bind_host.clone(), + ..light_client_prover_rollup.rpc + }, + runner: runner_config.clone(), + ..light_client_prover_rollup + } + }; + + let full_node_rollup = { + let bind_port = get_available_port()?; + let node_kind = NodeKind::FullNode.to_string(); + RollupConfig { + da: BitcoinServiceConfig { + node_url: format!( + "http://{}/wallet/{}", + da_config.node_url, + NodeKind::Bitcoin // Use default wallet + ), + tx_backup_dir: tx_backup_dir.display().to_string(), + ..da_config.clone() + }, + storage: StorageConfig { + path: dbs_dir.join(format!("{node_kind}-db")), + db_max_open_files: None, + }, + rpc: RpcConfig { + bind_port, + bind_host: bind_host.clone(), + ..full_node_rollup.rpc + }, + runner: runner_config.clone(), + ..full_node_rollup + } + }; + + Ok(TestConfig { + bitcoin: bitcoin_confs, + sequencer: FullSequencerConfig::new( + sequencer, + sequencer_rollup, + None, + sequencer_dir, + env.sequencer(), + )?, + batch_prover: FullBatchProverConfig::new( + batch_prover, + batch_prover_rollup, + None, + batch_prover_dir, + env.batch_prover(), + )?, + light_client_prover: FullLightClientProverConfig::new( + light_client_prover, + light_client_prover_rollup, + None, + light_client_prover_dir, + env.light_client_prover(), + )?, + full_node: FullFullNodeConfig::new( + (), + full_node_rollup, + None, + full_node_dir, + env.full_node(), + )?, + test_case, + }) } fn create_dirs(base_dir: &Path) -> Result<[PathBuf; 8]> { diff --git a/src/node.rs b/src/node.rs index 508b809..e16649e 100644 --- a/src/node.rs +++ b/src/node.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::{bail, Context}; use async_trait::async_trait; +use bitcoincore_rpc::{Auth, Client as BitcoinClient}; use serde::Serialize; use tokio::{ process::Command, @@ -25,7 +26,6 @@ use crate::{ utils::{get_citrea_path, get_genesis_path}, Result, }; -use bitcoincore_rpc::{Auth, Client as BitcoinClient}; #[derive(Debug, Clone, Eq, Hash, PartialEq)] pub enum NodeKind { diff --git a/src/test_case.rs b/src/test_case.rs index 82d017c..e1aa153 100644 --- a/src/test_case.rs +++ b/src/test_case.rs @@ -1,6 +1,18 @@ //! This module provides the `TestCaseRunner` and `TestCase` trait for running and defining test cases. //! It handles setup, execution, and cleanup of test environments. +use std::{ + io::Write, + panic::{self}, + path::Path, + time::Duration, +}; + +use anyhow::{bail, Context}; +use async_trait::async_trait; +use futures::FutureExt; +use tokio::signal; + use super::{ config::{BitcoinConfig, TestCaseConfig, TestCaseEnv}, framework::TestFramework, @@ -10,15 +22,6 @@ use crate::{ config::{BatchProverConfig, LightClientProverConfig, SequencerConfig}, traits::NodeT, }; -use anyhow::{bail, Context}; -use async_trait::async_trait; -use futures::FutureExt; -use std::{ - panic::{self}, - path::Path, - time::Duration, -}; -use tokio::signal; const CITREA_ENV: &str = "CITREA_E2E_TEST_BINARY"; const BITCOIN_ENV: &str = "BITCOIN_E2E_TEST_BINARY"; @@ -98,9 +101,11 @@ impl TestCaseRunner { .as_mut() .with_context(|| format!("Framework not correctly initialized, result {result:?}"))?; - if let Err(_) | Ok(Err(_)) = result { - if let Err(e) = f.dump_logs() { - eprintln!("Error dumping log: {e}"); + if std::env::var("DISABLE_DUMP_LOGS").is_err() { + if let Err(_) | Ok(Err(_)) = result { + if let Err(e) = f.dump_logs() { + eprintln!("Error dumping log: {e}"); + } } } @@ -109,6 +114,9 @@ impl TestCaseRunner { // Additional test cleanup self.0.cleanup().await?; + std::io::stdout().flush()?; + std::io::stderr().flush()?; + match result { Ok(Ok(())) => Ok(()), Ok(Err(e)) => Err(e), diff --git a/src/traits.rs b/src/traits.rs index 819e6b4..20c5b71 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -6,9 +6,8 @@ use bollard::{container::StopContainerOptions, Docker}; use tokio::process::Child; use tracing::info; -use crate::docker::{ContainerSpawnOutput, DockerEnv}; - use super::Result; +use crate::docker::{ContainerSpawnOutput, DockerEnv}; #[derive(Debug)] pub enum SpawnOutput { diff --git a/src/utils.rs b/src/utils.rs index 73442a7..a31c296 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,9 +8,8 @@ use std::{ use anyhow::anyhow; use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use crate::node::Config; - use super::Result; +use crate::node::Config; pub fn get_available_port() -> Result { let listener = TcpListener::bind("127.0.0.1:0")?; diff --git a/tests/mod.rs b/tests/mod.rs index f214fd7..3399098 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -1,10 +1,12 @@ use async_trait::async_trait; use bitcoincore_rpc::RpcApi; -use citrea_e2e::bitcoin::FINALITY_DEPTH; -use citrea_e2e::config::{TestCaseConfig, TestCaseDockerConfig}; -use citrea_e2e::framework::TestFramework; -use citrea_e2e::test_case::{TestCase, TestCaseRunner}; -use citrea_e2e::Result; +use citrea_e2e::{ + bitcoin::FINALITY_DEPTH, + config::{TestCaseConfig, TestCaseDockerConfig}, + framework::TestFramework, + test_case::{TestCase, TestCaseRunner}, + Result, +}; struct DockerIntegrationTest;