Merge pull request #85 from Fantom-foundation/rapol/feature/protobuf

[Protobuf] Working prototype

db/substate_db.go

@@ -4,9 +4,7 @@ import (
 	"encoding/binary"
 	"fmt"
 
-	"github.com/Fantom-foundation/Substate/rlp"
 	"github.com/Fantom-foundation/Substate/substate"
-	trlp "github.com/Fantom-foundation/Substate/types/rlp"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -63,11 +61,15 @@ func NewSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.Re
 }
 
 func MakeDefaultSubstateDB(db *leveldb.DB) SubstateDB {
-	return &substateDB{&codeDB{&baseDB{backend: db}}, nil}
+	sdb := &substateDB{&codeDB{&baseDB{backend: db}}, nil}
+	sdb, _ = sdb.SetSubstateEncoding("default")
+	return sdb
 }
 
 func MakeDefaultSubstateDBFromBaseDB(db BaseDB) SubstateDB {
-	return &substateDB{&codeDB{&baseDB{backend: db.getBackend()}}, nil}
+	sdb := &substateDB{&codeDB{&baseDB{backend: db.getBackend()}}, nil}
+	sdb, _ = sdb.SetSubstateEncoding("default")
+	return sdb
 }
 
 // NewReadOnlySubstateDB creates a new instance of read-only SubstateDB.
@@ -76,15 +78,20 @@ func NewReadOnlySubstateDB(path string) (SubstateDB, error) {
 }
 
 func MakeSubstateDB(db *leveldb.DB, wo *opt.WriteOptions, ro *opt.ReadOptions) SubstateDB {
-	return &substateDB{&codeDB{&baseDB{backend: db, wo: wo, ro: ro}}, nil}
+	sdb := &substateDB{&codeDB{&baseDB{backend: db, wo: wo, ro: ro}}, nil}
+	sdb, _ = sdb.SetSubstateEncoding("default")
+	return sdb
 }
 
 func newSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*substateDB, error) {
 	base, err := newCodeDB(path, o, wo, ro)
 	if err != nil {
 		return nil, err
 	}
-	return &substateDB{base, nil}, nil
+
+	sdb := &substateDB{base, nil}
+	sdb, _ = sdb.SetSubstateEncoding("default")
+	return sdb, nil
 }
 
 type substateDB struct {
@@ -180,10 +187,9 @@ func (db *substateDB) PutSubstate(ss *substate.Substate) error {
 
 	key := SubstateDBKey(ss.Block, ss.Transaction)
 
-	substateRLP := rlp.NewRLP(ss)
-	value, err := trlp.EncodeToBytes(substateRLP)
+	value, err := db.encodeSubstate(ss, ss.Block, ss.Transaction)
 	if err != nil {
-		return fmt.Errorf("cannot encode substate-rlp block %v, tx %v; %v", ss.Block, ss.Transaction, err)
+		return fmt.Errorf("cannot encode substate block %v, tx %v; %v", ss.Block, ss.Transaction, err)
 	}
 
 	return db.Put(key, value)

db/substate_db_test.go

@@ -24,7 +24,7 @@ var testSubstate = &substate.Substate{
 		BaseFee:    new(big.Int).SetUint64(1),
 	},
 	Message:     substate.NewMessage(1, true, new(big.Int).SetUint64(1), 1, types.Address{1}, new(types.Address), new(big.Int).SetUint64(1), []byte{1}, nil, types.AccessList{}, new(big.Int).SetUint64(1), new(big.Int).SetUint64(1), new(big.Int).SetUint64(1), make([]types.Hash, 0)),
-	Result:      substate.NewResult(1, types.Bloom{}, []*types.Log{}, types.Address{1}, 1),
+	Result:      substate.NewResult(1, types.Bloom{}, []*types.Log{}, types.Address{}, 1),
 	Block:       37_534_834,
 	Transaction: 1,
 }

db/substate_encoding.go

@@ -3,21 +3,24 @@ package db
 import (
 	"fmt"
 
+	pb "github.com/Fantom-foundation/Substate/protobuf"
 	"github.com/Fantom-foundation/Substate/rlp"
 	"github.com/Fantom-foundation/Substate/substate"
 	"github.com/Fantom-foundation/Substate/types"
+	trlp "github.com/Fantom-foundation/Substate/types/rlp"
+	"github.com/golang/protobuf/proto"
 )
 
 // SetSubstateEncoding sets the runtime encoding/decoding behavior of substateDB
 // intended usage:
 //
-//		db := &substateDB{..}
+//	db := &substateDB{..} // default to rlp
 //	     db, err := db.SetSubstateEncoding(<schema>) // set encoding
 //	     db.GetSubstateDecoder() // returns configured encoding
 func (db *substateDB) SetSubstateEncoding(schema string) (*substateDB, error) {
 	encoding, err := newSubstateEncoding(schema, db.GetCode)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to set decoder; %w", err)
+		return nil, fmt.Errorf("failed to set decoder; %w", err)
 	}
 
 	db.encoding = encoding
@@ -35,40 +38,56 @@ func (db *substateDB) GetSubstateEncoding() string {
 type substateEncoding struct {
 	schema string
 	decode decodeFunc
+	encode encodeFunc
 }
 
 // decodeFunc aliases the common function used to decode substate
 type decodeFunc func([]byte, uint64, int) (*substate.Substate, error)
 
+// encodeFunc alias the common function used to encode substate
+type encodeFunc func(*substate.Substate, uint64, int) ([]byte, error)
+
 // codeLookupFunc aliases codehash->code lookup necessary to decode substate
 type codeLookupFunc = func(types.Hash) ([]byte, error)
 
 // newSubstateDecoder returns requested SubstateDecoder
 func newSubstateEncoding(encoding string, lookup codeLookupFunc) (*substateEncoding, error) {
 	switch encoding {
 
-	case "default", "rlp":
+	case "", "default", "rlp":
 		return &substateEncoding{
 			schema: "rlp",
 			decode: func(bytes []byte, block uint64, tx int) (*substate.Substate, error) {
 				return decodeRlp(bytes, lookup, block, tx)
 			},
+			encode: encodeRlp,
+		}, nil
+
+	case "protobuf", "pb":
+		return &substateEncoding{
+			schema: "protobuf",
+			decode: func(bytes []byte, block uint64, tx int) (*substate.Substate, error) {
+				return decodeProtobuf(bytes, lookup, block, tx)
+			},
+			encode: pb.Encode,
 		}, nil
 
 	default:
-		return nil, fmt.Errorf("Encoding not supported: %s", encoding)
+		return nil, fmt.Errorf("encoding not supported: %s", encoding)
 
 	}
 }
 
 // decodeSubstate defensively defaults to "default" if nil
 func (db *substateDB) decodeToSubstate(bytes []byte, block uint64, tx int) (*substate.Substate, error) {
-	if db.encoding == nil {
-		db.SetSubstateEncoding("default")
-	}
 	return db.encoding.decode(bytes, block, tx)
 }
 
+// encodeSubstate defensively defaults to "default" if nil
+func (db *substateDB) encodeSubstate(ss *substate.Substate, block uint64, tx int) ([]byte, error) {
+	return db.encoding.encode(ss, block, tx)
+}
+
 // decodeRlp decodes into substate the provided rlp-encoded bytecode
 func decodeRlp(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*substate.Substate, error) {
 	rlpSubstate, err := rlp.Decode(bytes)
@@ -78,3 +97,23 @@ func decodeRlp(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*subs
 
 	return rlpSubstate.ToSubstate(lookup, block, tx)
 }
+
+// encodeRlp encodes substate into rlp-encoded bytes
+func encodeRlp(ss *substate.Substate, block uint64, tx int) ([]byte, error) {
+	bytes, err := trlp.EncodeToBytes(rlp.NewRLP(ss))
+	if err != nil {
+		return nil, fmt.Errorf("cannot encode substate into rlp block: %v, tx %v; %w", block, tx, err)
+	}
+
+	return bytes, nil
+}
+
+// decodeProtobuf decodes protobuf-encoded bytecode into substate
+func decodeProtobuf(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*substate.Substate, error) {
+	pbSubstate := &pb.Substate{}
+	if err := proto.Unmarshal(bytes, pbSubstate); err != nil {
+		return nil, fmt.Errorf("cannot decode substate data from protobuf block: %v, tx %v; %w", block, tx, err)
+	}
+
+	return pbSubstate.Decode(lookup, block, tx)
+}

db/substate_encoding_test.go

@@ -4,17 +4,30 @@ import (
 	"strings"
 	"testing"
 
+	pb "github.com/Fantom-foundation/Substate/protobuf"
 	"github.com/Fantom-foundation/Substate/rlp"
 	trlp "github.com/Fantom-foundation/Substate/types/rlp"
 )
 
+type encTest struct {
+	bytes []byte
+	blk   uint64
+	tx    int
+}
+
 var (
-	testRlp, _ = trlp.EncodeToBytes(rlp.NewRLP(testSubstate))
-	testBlk    = testSubstate.Block
-	testTx     = testSubstate.Transaction
+	blk = testSubstate.Block
+	tx  = testSubstate.Transaction
+
+	simplePb, _ = pb.Encode(testSubstate, blk, tx)
+	testPb      = encTest{bytes: simplePb, blk: blk, tx: tx}
 
-	supportedEncoding = map[string][]byte{
-		"rlp": testRlp,
+	simpleRlp, _ = trlp.EncodeToBytes(rlp.NewRLP(testSubstate))
+	testRlp      = encTest{bytes: simpleRlp, blk: blk, tx: tx}
+
+	supportedEncoding = map[string]encTest{
+		"rlp":      testRlp,
+		"protobuf": testPb,
 	}
 )
 
@@ -25,40 +38,41 @@ func TestSubstateEncoding_NilEncodingDefaultsToRlp(t *testing.T) {
 		t.Errorf("cannot open db; %v", err)
 	}
 
-	if got := db.GetSubstateEncoding(); got != "" {
+	// purposely never set encoding
+
+	// defaults to rlp
+	if got := db.GetSubstateEncoding(); got != "rlp" {
 		t.Fatalf("substate encoding should be nil, got: %s", got)
 	}
 
-	// purposely never set encoding
-	_, err = db.decodeToSubstate(testRlp, testBlk, testTx)
+	_, err = db.decodeToSubstate(testRlp.bytes, testRlp.blk, testRlp.tx)
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	if got := db.GetSubstateEncoding(); got != "rlp" {
-		t.Fatalf("db should default to rlp, got: %s", got)
-	}
 }
 
 func TestSubstateEncoding_DefaultEncodingDefaultsToRlp(t *testing.T) {
-	path := t.TempDir() + "test-db"
-	db, err := newSubstateDB(path, nil, nil, nil)
-	if err != nil {
-		t.Errorf("cannot open db; %v", err)
-	}
+	defaultKeywords := []string{"", "default"}
+	for _, defaultEncoding := range defaultKeywords {
+		path := t.TempDir() + "test-db-" + defaultEncoding
+		db, err := newSubstateDB(path, nil, nil, nil)
+		if err != nil {
+			t.Errorf("cannot open db; %v", err)
+		}
 
-	_, err = db.SetSubstateEncoding("default")
-	if err != nil {
-		t.Fatal("default is supportet, but error")
-	}
+		_, err = db.SetSubstateEncoding(defaultEncoding)
+		if err != nil {
+			t.Fatalf("Default encoding '%s' must be supported, but error", defaultEncoding)
+		}
 
-	_, err = db.decodeToSubstate(testRlp, testBlk, testTx)
-	if err != nil {
-		t.Fatal(err)
-	}
+		_, err = db.decodeToSubstate(testRlp.bytes, testRlp.blk, testRlp.tx)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	if got := db.GetSubstateEncoding(); got != "rlp" {
-		t.Fatalf("db should default to rlp, got: %s", got)
+		if got := db.GetSubstateEncoding(); got != "rlp" {
+			t.Fatalf("db should default to rlp, got: %s", got)
+		}
 	}
 }
 
@@ -70,30 +84,30 @@ func TestSubstateEncoding_UnsupportedEncodingThrowsError(t *testing.T) {
 	}
 
 	_, err = db.SetSubstateEncoding("EncodingNotSupported")
-	if err == nil || !strings.Contains(err.Error(), "Encoding not supported") {
-		t.Error("Encoding not supported, but no error")
+	if err == nil || !strings.Contains(err.Error(), "encoding not supported") {
+		t.Error("encoding not supported, but no error")
 	}
 }
 
 func TestSubstateEncoding_TestDb(t *testing.T) {
-	path := t.TempDir() + "test-db"
-	db, err := newSubstateDB(path, nil, nil, nil)
-	if err != nil {
-		t.Errorf("cannot open db; %v", err)
-	}
+	for encoding, et := range supportedEncoding {
+		path := t.TempDir() + "test-db-" + encoding
+		db, err := newSubstateDB(path, nil, nil, nil)
+		if err != nil {
+			t.Errorf("cannot open db; %v", err)
+		}
 
-	for encoding, bytes := range supportedEncoding {
-		_, err = db.SetSubstateEncoding(encoding)
+		db, err = db.SetSubstateEncoding(encoding)
 		if err != nil {
 			t.Error(err)
 		}
 
-		ss, err := db.decodeToSubstate(bytes, testBlk, testTx)
+		ss, err := db.decodeToSubstate(et.bytes, et.blk, et.tx)
 		if err != nil {
 			t.Error(err)
 		}
 
-		err = addCustomSubstate(db, testBlk, ss)
+		err = addCustomSubstate(db, et.blk, ss)
 		if err != nil {
 			t.Error(err)
 		}
@@ -103,24 +117,24 @@ func TestSubstateEncoding_TestDb(t *testing.T) {
 }
 
 func TestSubstateEncoding_TestIterator(t *testing.T) {
-	path := t.TempDir() + "test-db"
-	db, err := newSubstateDB(path, nil, nil, nil)
-	if err != nil {
-		t.Errorf("cannot open db; %v", err)
-	}
+	for encoding, et := range supportedEncoding {
+		path := t.TempDir() + "test-db-" + encoding
+		db, err := newSubstateDB(path, nil, nil, nil)
+		if err != nil {
+			t.Errorf("cannot open db; %v", err)
+		}
 
-	for encoding, bytes := range supportedEncoding {
 		_, err = db.SetSubstateEncoding(encoding)
 		if err != nil {
 			t.Error(err)
 		}
 
-		ss, err := db.decodeToSubstate(bytes, testBlk, testTx)
+		ss, err := db.decodeToSubstate(et.bytes, et.blk, et.tx)
 		if err != nil {
 			t.Error(err)
 		}
 
-		err = addCustomSubstate(db, testBlk, ss)
+		err = addCustomSubstate(db, et.blk, ss)
 		if err != nil {
 			t.Error(err)
 		}

go.mod

@@ -3,20 +3,24 @@ module github.com/Fantom-foundation/Substate
 go 1.21
 
 require (
-	github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954
-	github.com/urfave/cli/v2 v2.24.4
-	golang.org/x/crypto v0.17.0
+	github.com/golang/protobuf v1.5.4
+	github.com/stretchr/testify v1.9.0
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/urfave/cli/v2 v2.25.7
+	golang.org/x/crypto v0.22.0
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
+	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/holiman/uint256 v1.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )