Merge branch 'feat-rollup-relayer-codecv4' into feat-coordinator-add-…

…blob-bytes-in-batch-proving-task

common/forks/forks.go

@@ -23,19 +23,17 @@ func GetHardforkName(config *params.ChainConfig, blockHeight, blockTimestamp uin
 
 // GetCodecVersion returns the encoding codec version for the given block height and timestamp.
 // It determines the appropriate codec version based on the active hardfork.
-func GetCodecVersion(config *params.ChainConfig, blockHeight, blockTimestamp uint64, enableConditionalCompress bool) encoding.CodecVersion {
+func GetCodecVersion(config *params.ChainConfig, blockHeight, blockTimestamp uint64) encoding.CodecVersion {
 	if !config.IsBernoulli(new(big.Int).SetUint64(blockHeight)) {
 		return encoding.CodecV0
 	} else if !config.IsCurie(new(big.Int).SetUint64(blockHeight)) {
 		return encoding.CodecV1
 	} else if !config.IsDarwin(blockTimestamp) {
 		return encoding.CodecV2
+	} else if !config.IsDarwinV2(blockTimestamp) {
+		return encoding.CodecV3
 	} else {
-		if !enableConditionalCompress {
-			return encoding.CodecV3
-		} else {
-			return encoding.CodecV4
-		}
+		return encoding.CodecV4
 	}
 }
 

prover/src/coordinator_client.rs

@@ -14,6 +14,8 @@ use types::*;
 
 use crate::{config::Config, key_signer::KeySigner};
 
+pub use errors::ProofStatusNotOKError;
+
 pub struct CoordinatorClient<'a> {
     api: Api,
     token: Option<String>,

prover/src/coordinator_client/api.rs

@@ -1,4 +1,6 @@
-use super::types::*;
+use crate::{coordinator_client::ProofStatusNotOKError, types::ProofStatus};
+
+use super::{errors::*, types::*};
 use anyhow::{bail, Result};
 use core::time::Duration;
 use reqwest::{header::CONTENT_TYPE, Url};
@@ -76,7 +78,23 @@ impl Api {
         token: &String,
     ) -> Result<Response<SubmitProofResponseData>> {
         let method = "/coordinator/v1/submit_proof";
-        self.post_with_token(method, req, token).await
+        let response = self
+            .post_with_token::<SubmitProofRequest, Response<SubmitProofResponseData>>(
+                method, req, token,
+            )
+            .await?;
+
+        // when req's status already not ok, we mark the error returned from coordinator and will
+        // ignore it later.
+        if response.errcode == ErrorCode::ErrCoordinatorHandleZkProofFailure
+            && req.status != ProofStatus::Ok
+            && response
+                .errmsg
+                .contains("validator failure proof msg status not ok")
+        {
+            return Err(anyhow::anyhow!(ProofStatusNotOKError));
+        }
+        Ok(response)
     }
 
     async fn post_with_token<Req, Resp>(

prover/src/coordinator_client/errors.rs

@@ -1,4 +1,5 @@
 use serde::{Deserialize, Deserializer};
+use std::fmt;
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum ErrorCode {
@@ -51,3 +52,14 @@ impl<'de> Deserialize<'de> for ErrorCode {
         Ok(ErrorCode::from_i32(v))
     }
 }
+
+// ====================================================
+
+#[derive(Debug, Clone)]
+pub struct ProofStatusNotOKError;
+
+impl fmt::Display for ProofStatusNotOKError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "proof status not ok")
+    }
+}

prover/src/main.rs

@@ -37,7 +37,7 @@ struct Args {
     log_file: Option<String>,
 }
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {
+fn start() -> Result<()> {
     let args = Args::parse();
 
     if args.version {
@@ -76,3 +76,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
+
+fn main() {
+    let result = start();
+    if let Err(e) = result {
+        log::error!("main exit with error {:#}", e)
+    }
+}

prover/src/task_processor.rs

@@ -1,4 +1,4 @@
-use super::{prover::Prover, task_cache::TaskCache};
+use super::{coordinator_client::ProofStatusNotOKError, prover::Prover, task_cache::TaskCache};
 use anyhow::{Context, Result};
 use std::rc::Rc;
 
@@ -16,7 +16,11 @@ impl<'a> TaskProcessor<'a> {
         loop {
             log::info!("start a new round.");
             if let Err(err) = self.prove_and_submit() {
-                log::error!("encounter error: {:#}", err);
+                if err.is::<ProofStatusNotOKError>() {
+                    log::info!("proof status not ok, downgrade level to info.");
+                } else {
+                    log::error!("encounter error: {:#}", err);
+                }
             } else {
                 log::info!("prove & submit succeed.");
             }
@@ -54,11 +58,18 @@ impl<'a> TaskProcessor<'a> {
             );
             let result = match self.prover.prove_task(&task_wrapper.task) {
                 Ok(proof_detail) => self.prover.submit_proof(proof_detail, &task_wrapper.task),
-                Err(error) => self.prover.submit_error(
-                    &task_wrapper.task,
-                    super::types::ProofFailureType::NoPanic,
-                    error,
-                ),
+                Err(error) => {
+                    log::error!(
+                        "failed to prove task, id: {}, error: {:#}",
+                        &task_wrapper.task.id,
+                        error
+                    );
+                    self.prover.submit_error(
+                        &task_wrapper.task,
+                        super::types::ProofFailureType::NoPanic,
+                        error,
+                    )
+                }
             };
             return result;
         }

prover/src/zk_circuits_handler.rs

@@ -119,7 +119,6 @@ impl<'a> CircuitsHandlerProvider<'a> {
                 if let Some(handler) = &self.current_circuit {
                     Ok(handler.clone())
                 } else {
-                    log::error!("missing cached handler, there must be something wrong.");
                     bail!("missing cached handler, there must be something wrong.")
                 }
             }
@@ -136,7 +135,6 @@ impl<'a> CircuitsHandlerProvider<'a> {
                     self.current_circuit = Some(rc_handler.clone());
                     Ok(rc_handler)
                 } else {
-                    log::error!("missing builder, there must be something wrong.");
                     bail!("missing builder, there must be something wrong.")
                 }
             }

rollup/conf/config.json

@@ -66,7 +66,6 @@
       "l1_commit_gas_limit_multiplier": 1.2
     },
     "chunk_proposer_config": {
-      "enable_conditional_compress": false,
       "propose_interval_milliseconds": 100,
       "max_block_num_per_chunk": 100,
       "max_tx_num_per_chunk": 100,

rollup/go.mod

@@ -10,8 +10,8 @@ require (
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/holiman/uint256 v1.2.4
 	github.com/prometheus/client_golang v1.16.0
-	github.com/scroll-tech/da-codec v0.0.0-20240819100936-c6af3bbe7068
-	github.com/scroll-tech/go-ethereum v1.10.14-0.20240626125436-418bc6f728b6
+	github.com/scroll-tech/da-codec v0.1.1-0.20240819100936-c6af3bbe7068
+	github.com/scroll-tech/go-ethereum v1.10.14-0.20240821075135-bdd1b005d40f
 	github.com/smartystreets/goconvey v1.8.0
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.25.7

rollup/go.sum

@@ -236,10 +236,10 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/da-codec v0.0.0-20240819100936-c6af3bbe7068 h1:oVGwhg4cCq35B04eG/S4OBXDwXiFH7+LezuH2ZTRBPs=
-github.com/scroll-tech/da-codec v0.0.0-20240819100936-c6af3bbe7068/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
-github.com/scroll-tech/go-ethereum v1.10.14-0.20240626125436-418bc6f728b6 h1:Q8YyvrcPIcXQwE4ucm4bqmPh6TP6IB1GUTXripf2WyQ=
-github.com/scroll-tech/go-ethereum v1.10.14-0.20240626125436-418bc6f728b6/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ=
+github.com/scroll-tech/da-codec v0.1.1-0.20240819100936-c6af3bbe7068 h1:KyTp4aedcpjr/rbntrmlhUxjrDYu1Q02QDLaF5vqpxs=
+github.com/scroll-tech/da-codec v0.1.1-0.20240819100936-c6af3bbe7068/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
+github.com/scroll-tech/go-ethereum v1.10.14-0.20240821075135-bdd1b005d40f h1:0XhY20/Sh2UCroZqD4orK7eDElQD2XK4GLrTbPmUBpw=
+github.com/scroll-tech/go-ethereum v1.10.14-0.20240821075135-bdd1b005d40f/go.mod h1:jLTGZ5iL5T7g1BEWrQXVIR+wutJFDTVs/mCfjAlrhrA=
 github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE=
 github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=

rollup/internal/config/l2.go

@@ -37,7 +37,6 @@ type ChunkProposerConfig struct {
 	MaxRowConsumptionPerChunk       uint64  `json:"max_row_consumption_per_chunk"`
 	GasCostIncreaseMultiplier       float64 `json:"gas_cost_increase_multiplier"`
 	MaxUncompressedBatchBytesSize   uint64  `json:"max_uncompressed_batch_bytes_size"`
-	EnableConditionalCompress       bool    `json:"enable_conditional_compress"`
 }
 
 // BatchProposerConfig loads batch_proposer configuration items.

rollup/internal/controller/relayer/l2_relayer.go

@@ -201,7 +201,7 @@ func (r *Layer2Relayer) initializeGenesis() error {
 
 	err = r.db.Transaction(func(dbTX *gorm.DB) error {
 		var dbChunk *orm.Chunk
-		dbChunk, err = r.chunkOrm.InsertChunk(r.ctx, chunk, encoding.CodecV0, false, rutils.ChunkMetrics{}, dbTX)
+		dbChunk, err = r.chunkOrm.InsertChunk(r.ctx, chunk, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.ChunkMetrics{}, dbTX)
 		if err != nil {
 			return fmt.Errorf("failed to insert chunk: %v", err)
 		}
@@ -218,7 +218,7 @@ func (r *Layer2Relayer) initializeGenesis() error {
 		}
 
 		var dbBatch *orm.Batch
-		dbBatch, err = r.batchOrm.InsertBatch(r.ctx, batch, encoding.CodecV0, false, rutils.BatchMetrics{}, dbTX)
+		dbBatch, err = r.batchOrm.InsertBatch(r.ctx, batch, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{}, dbTX)
 		if err != nil {
 			return fmt.Errorf("failed to insert batch: %v", err)
 		}

rollup/internal/controller/relayer/l2_relayer_test.go

@@ -79,9 +79,9 @@ func testL2RelayerProcessPendingBatches(t *testing.T) {
 		err = l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2})
 		assert.NoError(t, err)
 		chunkOrm := orm.NewChunk(db)
-		_, err = chunkOrm.InsertChunk(context.Background(), chunk1, codecVersion, false, rutils.ChunkMetrics{})
+		_, err = chunkOrm.InsertChunk(context.Background(), chunk1, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
-		_, err = chunkOrm.InsertChunk(context.Background(), chunk2, codecVersion, false, rutils.ChunkMetrics{})
+		_, err = chunkOrm.InsertChunk(context.Background(), chunk2, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
 
 		batch := &encoding.Batch{
@@ -92,7 +92,7 @@ func testL2RelayerProcessPendingBatches(t *testing.T) {
 		}
 
 		batchOrm := orm.NewBatch(db)
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, codecVersion, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: codecVersion}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 
 		relayer.ProcessPendingBatches()
@@ -128,9 +128,9 @@ func testL2RelayerProcessCommittedBatches(t *testing.T) {
 		err = l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2})
 		assert.NoError(t, err)
 		chunkOrm := orm.NewChunk(db)
-		_, err = chunkOrm.InsertChunk(context.Background(), chunk1, codecVersion, false, rutils.ChunkMetrics{})
+		_, err = chunkOrm.InsertChunk(context.Background(), chunk1, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
-		_, err = chunkOrm.InsertChunk(context.Background(), chunk2, codecVersion, false, rutils.ChunkMetrics{})
+		_, err = chunkOrm.InsertChunk(context.Background(), chunk2, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
 
 		batch := &encoding.Batch{
@@ -141,7 +141,7 @@ func testL2RelayerProcessCommittedBatches(t *testing.T) {
 		}
 
 		batchOrm := orm.NewBatch(db)
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, codecVersion, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: codecVersion}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 
 		err = batchOrm.UpdateRollupStatus(context.Background(), dbBatch.Hash, types.RollupCommitted)
@@ -197,7 +197,7 @@ func testL2RelayerProcessPendingBundles(t *testing.T) {
 		}
 
 		batchOrm := orm.NewBatch(db)
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, codecVersion, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: codecVersion}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 
 		bundleOrm := orm.NewBundle(db)
@@ -259,9 +259,9 @@ func testL2RelayerFinalizeTimeoutBatches(t *testing.T) {
 		err = l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2})
 		assert.NoError(t, err)
 		chunkOrm := orm.NewChunk(db)
-		chunkDB1, err := chunkOrm.InsertChunk(context.Background(), chunk1, codecVersion, false, rutils.ChunkMetrics{})
+		chunkDB1, err := chunkOrm.InsertChunk(context.Background(), chunk1, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
-		chunkDB2, err := chunkOrm.InsertChunk(context.Background(), chunk2, codecVersion, false, rutils.ChunkMetrics{})
+		chunkDB2, err := chunkOrm.InsertChunk(context.Background(), chunk2, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
 
 		batch := &encoding.Batch{
@@ -272,7 +272,7 @@ func testL2RelayerFinalizeTimeoutBatches(t *testing.T) {
 		}
 
 		batchOrm := orm.NewBatch(db)
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, codecVersion, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: codecVersion}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 
 		err = batchOrm.UpdateRollupStatus(context.Background(), dbBatch.Hash, types.RollupCommitted)
@@ -326,9 +326,9 @@ func testL2RelayerFinalizeTimeoutBundles(t *testing.T) {
 		err = l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2})
 		assert.NoError(t, err)
 		chunkOrm := orm.NewChunk(db)
-		chunkDB1, err := chunkOrm.InsertChunk(context.Background(), chunk1, codecVersion, false, rutils.ChunkMetrics{})
+		chunkDB1, err := chunkOrm.InsertChunk(context.Background(), chunk1, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
-		chunkDB2, err := chunkOrm.InsertChunk(context.Background(), chunk2, codecVersion, false, rutils.ChunkMetrics{})
+		chunkDB2, err := chunkOrm.InsertChunk(context.Background(), chunk2, rutils.CodecConfig{Version: codecVersion}, rutils.ChunkMetrics{})
 		assert.NoError(t, err)
 
 		batch := &encoding.Batch{
@@ -339,7 +339,7 @@ func testL2RelayerFinalizeTimeoutBundles(t *testing.T) {
 		}
 
 		batchOrm := orm.NewBatch(db)
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, codecVersion, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: codecVersion}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 
 		err = batchOrm.UpdateRollupStatus(context.Background(), dbBatch.Hash, types.RollupCommitted)
@@ -411,7 +411,7 @@ func testL2RelayerCommitConfirm(t *testing.T) {
 			Chunks:                     []*encoding.Chunk{chunk1, chunk2},
 		}
 
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, encoding.CodecV0, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 		batchHashes[i] = dbBatch.Hash
 	}
@@ -467,7 +467,7 @@ func testL2RelayerFinalizeBatchConfirm(t *testing.T) {
 			Chunks:                     []*encoding.Chunk{chunk1, chunk2},
 		}
 
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, encoding.CodecV0, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 		batchHashes[i] = dbBatch.Hash
 	}
@@ -525,7 +525,7 @@ func testL2RelayerFinalizeBundleConfirm(t *testing.T) {
 			Chunks:                     []*encoding.Chunk{chunk1, chunk2},
 		}
 
-		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, encoding.CodecV0, false, rutils.BatchMetrics{})
+		dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{})
 		assert.NoError(t, err)
 		batchHashes[i] = dbBatch.Hash
 
@@ -580,7 +580,7 @@ func testL2RelayerGasOracleConfirm(t *testing.T) {
 	}
 
 	batchOrm := orm.NewBatch(db)
-	dbBatch1, err := batchOrm.InsertBatch(context.Background(), batch1, encoding.CodecV0, false, rutils.BatchMetrics{})
+	dbBatch1, err := batchOrm.InsertBatch(context.Background(), batch1, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{})
 	assert.NoError(t, err)
 
 	batch2 := &encoding.Batch{
@@ -590,7 +590,7 @@ func testL2RelayerGasOracleConfirm(t *testing.T) {
 		Chunks:                     []*encoding.Chunk{chunk2},
 	}
 
-	dbBatch2, err := batchOrm.InsertBatch(context.Background(), batch2, encoding.CodecV0, false, rutils.BatchMetrics{})
+	dbBatch2, err := batchOrm.InsertBatch(context.Background(), batch2, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{})
 	assert.NoError(t, err)
 
 	// Create and set up the Layer2 Relayer.
@@ -742,9 +742,9 @@ func testGetBatchStatusByIndex(t *testing.T) {
 	err = l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2})
 	assert.NoError(t, err)
 	chunkOrm := orm.NewChunk(db)
-	_, err = chunkOrm.InsertChunk(context.Background(), chunk1, encoding.CodecV0, false, rutils.ChunkMetrics{})
+	_, err = chunkOrm.InsertChunk(context.Background(), chunk1, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.ChunkMetrics{})
 	assert.NoError(t, err)
-	_, err = chunkOrm.InsertChunk(context.Background(), chunk2, encoding.CodecV0, false, rutils.ChunkMetrics{})
+	_, err = chunkOrm.InsertChunk(context.Background(), chunk2, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.ChunkMetrics{})
 	assert.NoError(t, err)
 
 	batch := &encoding.Batch{
@@ -755,7 +755,7 @@ func testGetBatchStatusByIndex(t *testing.T) {
 	}
 
 	batchOrm := orm.NewBatch(db)
-	dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, encoding.CodecV0, false, rutils.BatchMetrics{})
+	dbBatch, err := batchOrm.InsertBatch(context.Background(), batch, rutils.CodecConfig{Version: encoding.CodecV0}, rutils.BatchMetrics{})
 	assert.NoError(t, err)
 
 	status, err := relayer.getBatchStatusByIndex(dbBatch)