[Protobuf] Working prototype

db/substate_db.go

@@ -4,9 +4,7 @@ import (
 	"encoding/binary"
 	"fmt"
 
-	"github.com/Fantom-foundation/Substate/rlp"
 	"github.com/Fantom-foundation/Substate/substate"
-	trlp "github.com/Fantom-foundation/Substate/types/rlp"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -180,10 +178,9 @@ func (db *substateDB) PutSubstate(ss *substate.Substate) error {
 
 	key := SubstateDBKey(ss.Block, ss.Transaction)
 
-	substateRLP := rlp.NewRLP(ss)
-	value, err := trlp.EncodeToBytes(substateRLP)
+	value, err := db.encodeSubstate(ss, ss.Block, ss.Transaction)
 	if err != nil {
-		return fmt.Errorf("cannot encode substate-rlp block %v, tx %v; %v", ss.Block, ss.Transaction, err)
+		return fmt.Errorf("cannot encode substate block %v, tx %v; %v", ss.Block, ss.Transaction, err)
 	}
 
 	return db.Put(key, value)

db/substate_db_test.go

@@ -24,7 +24,7 @@ var testSubstate = &substate.Substate{
 		BaseFee:    new(big.Int).SetUint64(1),
 	},
 	Message:     substate.NewMessage(1, true, new(big.Int).SetUint64(1), 1, types.Address{1}, new(types.Address), new(big.Int).SetUint64(1), []byte{1}, nil, types.AccessList{}, new(big.Int).SetUint64(1), new(big.Int).SetUint64(1), new(big.Int).SetUint64(1), make([]types.Hash, 0)),
-	Result:      substate.NewResult(1, types.Bloom{}, []*types.Log{}, types.Address{1}, 1),
+	Result:      substate.NewResult(1, types.Bloom{}, []*types.Log{}, types.Address{}, 1),
 	Block:       37_534_834,
 	Transaction: 1,
 }

db/substate_encoding.go

@@ -3,9 +3,12 @@ package db
 import (
 	"fmt"
 
+	pb "github.com/Fantom-foundation/Substate/protobuf"
 	"github.com/Fantom-foundation/Substate/rlp"
 	"github.com/Fantom-foundation/Substate/substate"
 	"github.com/Fantom-foundation/Substate/types"
+	trlp "github.com/Fantom-foundation/Substate/types/rlp"
+	"github.com/golang/protobuf/proto"
 )
 
 // SetSubstateEncoding sets the runtime encoding/decoding behavior of substateDB
@@ -35,24 +38,38 @@ func (db *substateDB) GetSubstateEncoding() string {
 type substateEncoding struct {
 	schema string
 	decode decodeFunc
+	encode encodeFunc
 }
 
 // decodeFunc aliases the common function used to decode substate
 type decodeFunc func([]byte, uint64, int) (*substate.Substate, error)
 
+// encodeFunc alias the common function used to encode substate
+type encodeFunc func(*substate.Substate, uint64, int) ([]byte, error)
+
 // codeLookupFunc aliases codehash->code lookup necessary to decode substate
 type codeLookupFunc = func(types.Hash) ([]byte, error)
 
 // newSubstateDecoder returns requested SubstateDecoder
 func newSubstateEncoding(encoding string, lookup codeLookupFunc) (*substateEncoding, error) {
 	switch encoding {
 
-	case "default", "rlp":
+	case "", "default", "rlp":
 		return &substateEncoding{
 			schema: "rlp",
 			decode: func(bytes []byte, block uint64, tx int) (*substate.Substate, error) {
 				return decodeRlp(bytes, lookup, block, tx)
 			},
+			encode: encodeRlp,
+		}, nil
+
+	case "protobuf", "pb":
+		return &substateEncoding{
+			schema: "protobuf",
+			decode: func(bytes []byte, block uint64, tx int) (*substate.Substate, error) {
+				return decodeProtobuf(bytes, lookup, block, tx)
+			},
+			encode: encodeProtobuf,
 		}, nil
 
 	default:
@@ -69,6 +86,14 @@ func (db *substateDB) decodeToSubstate(bytes []byte, block uint64, tx int) (*sub
 	return db.encoding.decode(bytes, block, tx)
 }
 
+// encodeSubstate defensively defaults to "default" if nil
+func (db *substateDB) encodeSubstate(ss *substate.Substate, block uint64, tx int) ([]byte, error) {
+	if db.encoding == nil {
+		db.SetSubstateEncoding("default")
+	}
+	return db.encoding.encode(ss, block, tx)
+}
+
 // decodeRlp decodes into substate the provided rlp-encoded bytecode
 func decodeRlp(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*substate.Substate, error) {
 	rlpSubstate, err := rlp.Decode(bytes)
@@ -78,3 +103,33 @@ func decodeRlp(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*subs
 
 	return rlpSubstate.ToSubstate(lookup, block, tx)
 }
+
+// encodeRlp encodes into rlp-encoded bytes the provided substate
+func encodeRlp(ss *substate.Substate, block uint64, tx int) ([]byte, error) {
+	bytes, err := trlp.EncodeToBytes(rlp.NewRLP(ss))
+	if err != nil {
+		return nil, fmt.Errorf("cannot encode substate into rlp block: %v, tx %v; %w", block, tx, err)
+	}
+
+	return bytes, nil
+}
+
+// decodeProtobuf decodes into substate the provided rlp-encoded bytecode
+func decodeProtobuf(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*substate.Substate, error) {
+	pbSubstate := &pb.Substate{}
+	if err := proto.Unmarshal(bytes, pbSubstate); err != nil {
+		return nil, fmt.Errorf("cannot decode substate data from protobuf block: %v, tx %v; %w", block, tx, err)
+	}
+
+	return pbSubstate.Decode(lookup, block, tx)
+}
+
+// encodeRlp encodes into rlp-encoded bytes the provided substate
+func encodeProtobuf(ss *substate.Substate, block uint64, tx int) ([]byte, error) {
+	bytes, err := proto.Marshal(pb.Encode(ss))
+	if err != nil {
+		return nil, fmt.Errorf("cannot encode substate into protobuf block: %v, tx %v; %w", block, tx, err)
+	}
+
+	return bytes, nil
+}

db/substate_encoding_test.go

@@ -4,17 +4,36 @@ import (
 	"strings"
 	"testing"
 
+	pb "github.com/Fantom-foundation/Substate/protobuf"
 	"github.com/Fantom-foundation/Substate/rlp"
 	trlp "github.com/Fantom-foundation/Substate/types/rlp"
+	"google.golang.org/protobuf/proto"
 )
 
+type encTest struct {
+	bytes []byte
+	blk   uint64
+	tx    int
+}
+
 var (
-	testRlp, _ = trlp.EncodeToBytes(rlp.NewRLP(testSubstate))
-	testBlk    = testSubstate.Block
-	testTx     = testSubstate.Transaction
+	simplePb, _ = proto.Marshal(pb.Encode(testSubstate))
+	testPb      = encTest{
+		bytes: simplePb,
+		blk:   testSubstate.Block,
+		tx:    testSubstate.Transaction,
+	}
+
+	simpleRlp, _ = trlp.EncodeToBytes(rlp.NewRLP(testSubstate))
+	testRlp      = encTest{
+		bytes: simpleRlp,
+		blk:   testSubstate.Block,
+		tx:    testSubstate.Transaction,
+	}
 
-	supportedEncoding = map[string][]byte{
-		"rlp": testRlp,
+	supportedEncoding = map[string]encTest{
+		"rlp":      testRlp,
+		"protobuf": testPb,
 	}
 )
 
@@ -30,7 +49,7 @@ func TestSubstateEncoding_NilEncodingDefaultsToRlp(t *testing.T) {
 	}
 
 	// purposely never set encoding
-	_, err = db.decodeToSubstate(testRlp, testBlk, testTx)
+	_, err = db.decodeToSubstate(testRlp.bytes, testRlp.blk, testRlp.tx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -41,24 +60,27 @@ func TestSubstateEncoding_NilEncodingDefaultsToRlp(t *testing.T) {
 }
 
 func TestSubstateEncoding_DefaultEncodingDefaultsToRlp(t *testing.T) {
-	path := t.TempDir() + "test-db"
-	db, err := newSubstateDB(path, nil, nil, nil)
-	if err != nil {
-		t.Errorf("cannot open db; %v", err)
-	}
+	defaultKeywords := []string{"", "default"}
+	for _, defaultEncoding := range defaultKeywords {
+		path := t.TempDir() + "test-db-" + defaultEncoding
+		db, err := newSubstateDB(path, nil, nil, nil)
+		if err != nil {
+			t.Errorf("cannot open db; %v", err)
+		}
 
-	_, err = db.SetSubstateEncoding("default")
-	if err != nil {
-		t.Fatal("default is supportet, but error")
-	}
+		_, err = db.SetSubstateEncoding(defaultEncoding)
+		if err != nil {
+			t.Fatalf("Default encoding '%s' must be supported, but error", defaultEncoding)
+		}
 
-	_, err = db.decodeToSubstate(testRlp, testBlk, testTx)
-	if err != nil {
-		t.Fatal(err)
-	}
+		_, err = db.decodeToSubstate(testRlp.bytes, testRlp.blk, testRlp.tx)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	if got := db.GetSubstateEncoding(); got != "rlp" {
-		t.Fatalf("db should default to rlp, got: %s", got)
+		if got := db.GetSubstateEncoding(); got != "rlp" {
+			t.Fatalf("db should default to rlp, got: %s", got)
+		}
 	}
 }
 
@@ -76,24 +98,24 @@ func TestSubstateEncoding_UnsupportedEncodingThrowsError(t *testing.T) {
 }
 
 func TestSubstateEncoding_TestDb(t *testing.T) {
-	path := t.TempDir() + "test-db"
-	db, err := newSubstateDB(path, nil, nil, nil)
-	if err != nil {
-		t.Errorf("cannot open db; %v", err)
-	}
+	for encoding, et := range supportedEncoding {
+		path := t.TempDir() + "test-db-" + encoding
+		db, err := newSubstateDB(path, nil, nil, nil)
+		if err != nil {
+			t.Errorf("cannot open db; %v", err)
+		}
 
-	for encoding, bytes := range supportedEncoding {
-		_, err = db.SetSubstateEncoding(encoding)
+		db, err = db.SetSubstateEncoding(encoding)
 		if err != nil {
 			t.Error(err)
 		}
 
-		ss, err := db.decodeToSubstate(bytes, testBlk, testTx)
+		ss, err := db.decodeToSubstate(et.bytes, et.blk, et.tx)
 		if err != nil {
 			t.Error(err)
 		}
 
-		err = addCustomSubstate(db, testBlk, ss)
+		err = addCustomSubstate(db, et.blk, ss)
 		if err != nil {
 			t.Error(err)
 		}
@@ -103,24 +125,24 @@ func TestSubstateEncoding_TestDb(t *testing.T) {
 }
 
 func TestSubstateEncoding_TestIterator(t *testing.T) {
-	path := t.TempDir() + "test-db"
-	db, err := newSubstateDB(path, nil, nil, nil)
-	if err != nil {
-		t.Errorf("cannot open db; %v", err)
-	}
+	for encoding, et := range supportedEncoding {
+		path := t.TempDir() + "test-db-" + encoding
+		db, err := newSubstateDB(path, nil, nil, nil)
+		if err != nil {
+			t.Errorf("cannot open db; %v", err)
+		}
 
-	for encoding, bytes := range supportedEncoding {
 		_, err = db.SetSubstateEncoding(encoding)
 		if err != nil {
 			t.Error(err)
 		}
 
-		ss, err := db.decodeToSubstate(bytes, testBlk, testTx)
+		ss, err := db.decodeToSubstate(et.bytes, et.blk, et.tx)
 		if err != nil {
 			t.Error(err)
 		}
 
-		err = addCustomSubstate(db, testBlk, ss)
+		err = addCustomSubstate(db, et.blk, ss)
 		if err != nil {
 			t.Error(err)
 		}

go.mod

@@ -1,22 +1,29 @@
 module github.com/Fantom-foundation/Substate
 
-go 1.21
+go 1.22
+
+toolchain go1.22.4
 
 require (
-	github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954
-	github.com/urfave/cli/v2 v2.24.4
-	golang.org/x/crypto v0.17.0
+	github.com/ethereum/go-ethereum v1.14.9
+	github.com/golang/protobuf v1.5.4
+	github.com/holiman/uint256 v1.3.1
+	github.com/stretchr/testify v1.9.0
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/urfave/cli/v2 v2.25.7
+	golang.org/x/crypto v0.22.0
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
+	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )