Add events and extrinsics parsing logic based on metadata (#326)

* events: Add initial metadata parsing logic

* events: Add override decoding funcs, add support for primitive arrays

* events: Extract decode type def

* events: Add event parsing test script, provide more details in error messages

* events: Add variant support for vectors

* milestone 1: Add type registry and test (#327)

* milestone: Add type registry and test

* milestone1: Add metadatas from multiple chains and docker file for tests

* registry: Add methods to create call and error registries

* dockerfile: Fix test Dockerfile

* registry: Add more tests

* Events parsing v2 milestone 2 (#338)

* milestone: Add type registry and test

* milestone1: Add metadatas from multiple chains and docker file for tests

* events: Add decoders for each event field type

* registry: Add decoders

* registry: Add retryable executor and use it in parser

* registry: Add mocks and more tests

* registry: Use BitVec when parsing bit sequences

* test: Add more mocks and tests

* registry: Update field name retrieval

* registry-test: Get headers instead of blocks

* Add generic chain RPC, more tests, and Dockerfiles for the 2nd milestone

* make: Add container name

* chain: Add constructor for default chain

* registry: Change field separator to

* parser: Add DefaultExtrinsicParser and DefaultExtrinsic

* retriever: Add DefaultExtrinsicRetriever and adjust tests

* rpc: Add more comments to default entities

* retriever: Enable all live tests

* Add registry readme (#351)

* Fix docker run for w3f milestone 2 tests (#353)

* Address review milestone 2 (#355)

* add configurable event threshold

* Add natural language desc on event parsing

* make: Remove grant related targets

---------

Co-authored-by: Miguel Hervas <miguel.hervas.lazaro@gmail.com>
Co-authored-by: Nikhil Ranjan <niklabh811@gmail.com>

README.md

@@ -17,6 +17,8 @@ This package is feature complete, but it is relatively new and might still conta
 
 Please refer to https://godoc.org/github.com/centrifuge/go-substrate-rpc-client
 
+### Usage test examples of Dynamic Parsing of events & extrinsics
+[Registry docs](registry/REGISTRY.md)
 ## Contributing
 
 1. Install dependencies by running `make`

error/error.go

@@ -0,0 +1,26 @@
+package error
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Error string
+
+func (e Error) Error() string {
+	return string(e)
+}
+
+func (e Error) Is(err error) bool {
+	return strings.Contains(string(e), err.Error())
+}
+
+func (e Error) Wrap(err error) Error {
+	return Error(fmt.Errorf("%s: %w", e, err).Error())
+}
+
+func (e Error) WithMsg(msgFormat string, formatArgs ...any) Error {
+	msg := fmt.Sprintf(msgFormat, formatArgs...)
+
+	return Error(fmt.Sprintf("%s: %s", e, msg))
+}

error/error_test.go

@@ -0,0 +1,36 @@
+package error
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testErr = Error("test error")
+)
+
+func TestError(t *testing.T) {
+	newStdErr := errors.New("new std error")
+	wrappedErr := testErr.Wrap(newStdErr)
+
+	assert.True(t, errors.Is(wrappedErr, testErr))
+	assert.True(t, errors.Is(wrappedErr, newStdErr))
+	assert.Equal(t, fmt.Sprintf("%s: %s", testErr.Error(), newStdErr.Error()), wrappedErr.Error())
+
+	newErr := Error("new error")
+	newWrappedErr := newErr.Wrap(wrappedErr)
+
+	assert.True(t, errors.Is(newWrappedErr, newErr))
+	assert.True(t, errors.Is(newWrappedErr, testErr))
+	assert.True(t, errors.Is(newWrappedErr, newStdErr))
+	assert.Equal(t, fmt.Sprintf("%s: %s", newErr.Error(), wrappedErr.Error()), newWrappedErr.Error())
+
+	err := testErr.WithMsg("%d", 1)
+	assert.Equal(t, fmt.Sprintf("%s: 1", testErr), err.Error())
+
+	err = testErr.WithMsg("test msg")
+	assert.Equal(t, fmt.Sprintf("%s: test msg", testErr), err.Error())
+}

registry/REGISTRY.md

@@ -0,0 +1,83 @@
+# GSRPC Registry
+The GSRPC Registry can parse target metadata information into an in-memory registry of complex structures. 
+
+By leveraging the on-chain metadata, GSRPC is more robust to changes on types, allowing clients to only keep updated the types that are relevant to their business operation.
+
+This registry can be used afterwards to decode data read from live chains (events & extrinsics).
+
+## How to parse events and its types
+First we instantiate the API with the client node and open a connection: 
+```go
+testURL := "wss://fullnode.parachain.centrifuge.io" // Your endpoint
+api, err := gsrpc.NewSubstrateAPI(testURL)
+
+if err != nil {
+    log.Printf("Couldn't connect to '%s': %s\n", testURL, err)
+    return
+}
+```
+Then we instantiate the Event Retriever logic which internally creates a new EventRegistry reading from the target metadata of the connected chain. We pass as well the state RPC so the storage API is available: 
+```go
+retriever, err := NewDefaultEventRetriever(state.NewEventProvider(api.RPC.State), api.RPC.State)
+
+if err != nil {
+    log.Printf("Couldn't create event retriever: %s", err)
+    return
+}
+```
+At this point what we need is a block hash to read the events within. In this example we get the latest block header and the correspondent block hash out of the block number:
+```go
+header, err := api.RPC.Chain.GetHeaderLatest()
+
+if err != nil {
+    log.Printf("Couldn't get latest header for '%s': %s\n", testURL, err)
+    return
+}
+
+blockHash, err := api.RPC.Chain.GetBlockHash(uint64(header.Number))
+
+if err != nil {
+    log.Printf("Couldn't retrieve blockHash for '%s', block number %d: %s\n", testURL, header.Number, err)
+    return
+}
+```
+Finally, we just use the retriever function to read all the events in that block based on the chain metadata loaded in the event registry: 
+```go
+events, err := retriever.GetEvents(blockHash)
+
+if err != nil {
+    log.Printf("Couldn't retrieve events for '%s', block number %d: %s\n", testURL, header.Number, err)
+    return
+}
+
+log.Printf("Found %d events for '%s', at block number %d.\n", len(events), testURL, header.Number)
+
+// Example of the events returned structure
+for _, event := range events {
+    log.Printf("Event ID: %x \n", event.EventID)
+    log.Printf("Event Name: %s \n", event.Name)
+    log.Printf("Event Fields Count: %d \n", len(event.Fields))
+    for k, v := range event.Fields {
+        log.Printf("Field Name: %s \n", k)
+        log.Printf("Field Type: %v \n", reflect.TypeOf(v))
+        log.Printf("Field Value: %v \n", v)
+    }
+}
+
+```
+
+## Extended Usage
+Since docs get outdated fairly quick, here are links to tests that will always be up-to-date.
+### Populate Call, Error & Events Registries
+[Browse me](registry_test.go)
+
+### Event retriever
+[TestLive_EventRetriever_GetEvents](retriever/event_retriever_live_test.go)
+### Extrinsic retriever
+Since chain runtimes can be customized, modifying core types such as Accounts, Signature payloads or Payment payloads, the code supports a customizable way of passing those custom types to the extrinsic retriever.
+
+On the other hand, since a great majority of chains do not need to change these types, the tool provides a default for the most common used ones.
+#### Using Chain Defaults
+[TestExtrinsicRetriever_NewDefault](retriever/extrinsic_retriever_test.go#L179)
+#### Using Custom core types
+[TestLive_ExtrinsicRetriever_GetExtrinsics](retriever/extrinsic_retriever_live_test.go)

registry/error.go

@@ -0,0 +1,51 @@
+package registry
+
+import libErr "github.com/centrifuge/go-substrate-rpc-client/v4/error"
+
+const (
+	ErrRecursiveDecodersResolving            = libErr.Error("recursive decoders resolving")
+	ErrErrorsTypeNotFound                    = libErr.Error("errors type not found")
+	ErrErrorsTypeNotVariant                  = libErr.Error("errors type not a variant")
+	ErrErrorFieldsRetrieval                  = libErr.Error("error fields retrieval")
+	ErrCallsTypeNotFound                     = libErr.Error("calls type not found")
+	ErrCallsTypeNotVariant                   = libErr.Error("calls type not a variant")
+	ErrCallFieldsRetrieval                   = libErr.Error("call fields retrieval")
+	ErrEventsTypeNotFound                    = libErr.Error("events type not found")
+	ErrEventsTypeNotVariant                  = libErr.Error("events type not a variant")
+	ErrEventFieldsRetrieval                  = libErr.Error("event fields retrieval")
+	ErrFieldDecoderForRecursiveFieldNotFound = libErr.Error("field decoder for recursive field not found")
+	ErrRecursiveFieldResolving               = libErr.Error("recursive field resolving")
+	ErrFieldTypeNotFound                     = libErr.Error("field type not found")
+	ErrFieldDecoderRetrieval                 = libErr.Error("field decoder retrieval")
+	ErrCompactFieldTypeNotFound              = libErr.Error("compact field type not found")
+	ErrCompositeTypeFieldsRetrieval          = libErr.Error("composite type fields retrieval")
+	ErrArrayFieldTypeNotFound                = libErr.Error("array field type not found")
+	ErrVectorFieldTypeNotFound               = libErr.Error("vector field type not found")
+	ErrFieldTypeDefinitionNotSupported       = libErr.Error("field type definition not supported")
+	ErrVariantTypeFieldsRetrieval            = libErr.Error("variant type fields decoding")
+	ErrCompactTupleItemTypeNotFound          = libErr.Error("compact tuple item type not found")
+	ErrCompactTupleItemFieldDecoderRetrieval = libErr.Error("compact tuple item field decoder retrieval")
+	ErrCompactCompositeFieldTypeNotFound     = libErr.Error("compact composite field type not found")
+	ErrCompactCompositeFieldDecoderRetrieval = libErr.Error("compact composite field decoder retrieval")
+	ErrArrayItemFieldDecoderRetrieval        = libErr.Error("array item field decoder retrieval")
+	ErrSliceItemFieldDecoderRetrieval        = libErr.Error("slice item field decoder retrieval")
+	ErrTupleItemTypeNotFound                 = libErr.Error("tuple item type not found")
+	ErrTupleItemFieldDecoderRetrieval        = libErr.Error("tuple item field decoder retrieval")
+	ErrBitStoreTypeNotFound                  = libErr.Error("bit store type not found")
+	ErrBitStoreTypeNotSupported              = libErr.Error("bit store type not supported")
+	ErrBitOrderTypeNotFound                  = libErr.Error("bit order type not found")
+	ErrBitOrderCreation                      = libErr.Error("bit order creation")
+	ErrPrimitiveTypeNotSupported             = libErr.Error("primitive type not supported")
+	ErrTypeFieldDecoding                     = libErr.Error("type field decoding")
+	ErrVariantByteDecoding                   = libErr.Error("variant byte decoding")
+	ErrVariantFieldDecoderNotFound           = libErr.Error("variant field decoder not found")
+	ErrArrayItemDecoderNotFound              = libErr.Error("array item decoder not found")
+	ErrArrayItemDecoding                     = libErr.Error("array item decoding")
+	ErrSliceItemDecoderNotFound              = libErr.Error("slice item decoder not found")
+	ErrSliceLengthDecoding                   = libErr.Error("slice length decoding")
+	ErrSliceItemDecoding                     = libErr.Error("slice item decoding")
+	ErrCompositeFieldDecoding                = libErr.Error("composite field decoding")
+	ErrValueDecoding                         = libErr.Error("value decoding")
+	ErrRecursiveFieldDecoderNotFound         = libErr.Error("recursive field decoder not found")
+	ErrBitVecDecoding                        = libErr.Error("bit vec decoding")
+)

registry/exec/exec.go

@@ -0,0 +1,161 @@
+package exec
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+//go:generate mockery --name RetryableExecutor --structname RetryableExecutorMock --filename exec_mock.go --inpackage
+
+// RetryableExecutor is the interface used for executing a closure and its fallback if the initial execution fails.
+//
+// The interface is generic over type T which represents the return value of the closure.
+type RetryableExecutor[T any] interface {
+	ExecWithFallback(execFn func() (T, error), fallbackFn func() error) (T, error)
+}
+
+// retryableExecutor implements RetryableExecutor.
+//
+// It can be configured via the provided OptsFn(s).
+type retryableExecutor[T any] struct {
+	opts *Opts
+}
+
+// NewRetryableExecutor creates a new RetryableExecutor.
+func NewRetryableExecutor[T any](opts ...OptsFn) RetryableExecutor[T] {
+	execOpts := NewDefaultExecOpts()
+
+	for _, opt := range opts {
+		opt(execOpts)
+	}
+
+	return &retryableExecutor[T]{
+		execOpts,
+	}
+}
+
+// ExecWithFallback will attempt to execute the provided execFn and, in the case of failure, it will execute
+// the fallbackFn and retry execution of execFn.
+func (r *retryableExecutor[T]) ExecWithFallback(execFn func() (T, error), fallbackFn func() error) (res T, err error) {
+	if execFn == nil {
+		return res, ErrMissingExecFn
+	}
+
+	if fallbackFn == nil {
+		return res, ErrMissingFallbackFn
+	}
+
+	execErr := &Error{}
+
+	retryCount := uint(0)
+
+	for {
+		res, err = execFn()
+
+		if err == nil {
+			return res, nil
+		}
+
+		execErr.AddErr(fmt.Errorf("exec function error: %w", err))
+
+		if retryCount == r.opts.maxRetryCount {
+			return res, execErr
+		}
+
+		if err = fallbackFn(); err != nil && !r.opts.retryOnFallbackError {
+			execErr.AddErr(fmt.Errorf("fallback function error: %w", err))
+
+			return res, execErr
+		}
+
+		retryCount++
+
+		time.Sleep(r.opts.retryTimeout)
+	}
+}
+
+var (
+	ErrMissingExecFn     = errors.New("no exec function provided")
+	ErrMissingFallbackFn = errors.New("no fallback function provided")
+)
+
+const (
+	defaultMaxRetryCount        = 3
+	defaultErrTimeout           = 0 * time.Second
+	defaultRetryOnFallbackError = true
+)
+
+// Opts holds the configurable options for a RetryableExecutor.
+type Opts struct {
+	// maxRetryCount holds maximum number of retries in the case of failure.
+	maxRetryCount uint
+
+	// retryTimeout holds the timeout between retries.
+	retryTimeout time.Duration
+
+	// retryOnFallbackError specifies whether a retry will be done in the case of
+	// failure of the fallback function.
+	retryOnFallbackError bool
+}
+
+// NewDefaultExecOpts creates the default Opts.
+func NewDefaultExecOpts() *Opts {
+	return &Opts{
+		maxRetryCount:        defaultMaxRetryCount,
+		retryTimeout:         defaultErrTimeout,
+		retryOnFallbackError: defaultRetryOnFallbackError,
+	}
+}
+
+// OptsFn is function that operate on Opts.
+type OptsFn func(opts *Opts)
+
+// WithMaxRetryCount sets the max retry count.
+//
+// Note that a default value is provided if the provided count is 0.
+func WithMaxRetryCount(maxRetryCount uint) OptsFn {
+	return func(opts *Opts) {
+		if maxRetryCount == 0 {
+			maxRetryCount = defaultMaxRetryCount
+		}
+
+		opts.maxRetryCount = maxRetryCount
+	}
+}
+
+// WithRetryTimeout sets the retry timeout.
+func WithRetryTimeout(retryTimeout time.Duration) OptsFn {
+	return func(opts *Opts) {
+		opts.retryTimeout = retryTimeout
+	}
+}
+
+// WithRetryOnFallBackError sets the retryOnFallbackError flag.
+func WithRetryOnFallBackError(retryOnFallbackError bool) OptsFn {
+	return func(opts *Opts) {
+		opts.retryOnFallbackError = retryOnFallbackError
+	}
+}
+
+// Error holds none or multiple errors that can happen during execution.
+type Error struct {
+	errs []error
+}
+
+// AddErr appends an error to the error slice of Error.
+func (e *Error) AddErr(err error) {
+	e.errs = append(e.errs, err)
+}
+
+// Error implements the standard error interface.
+func (e *Error) Error() string {
+	sb := strings.Builder{}
+
+	for i, err := range e.errs {
+		sb.WriteString(fmt.Sprintf("error %d: %s\n", i, err))
+	}
+
+	return sb.String()
+}