feat(observability):add ready&health check api for k8s (#938)

Co-authored-by: georgehao <georgehao@users.noreply.github.com>
Co-authored-by: Péter Garamvölgyi <peter@scroll.io>

bridge-history-api/internal/controller/batch_controller.go

@@ -24,14 +24,14 @@ func NewBatchController(db *gorm.DB) *BatchController {
 func (b *BatchController) GetWithdrawRootByBatchIndex(ctx *gin.Context) {
 	var req types.QueryByBatchIndexRequest
 	if err := ctx.ShouldBind(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, err)
 		return
 	}
 	result, err := b.batchLogic.GetWithdrawRootByBatchIndex(ctx, req.BatchIndex)
 	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetWithdrawRootByBatchIndexFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrGetWithdrawRootByBatchIndexFailure, err)
 		return
 	}
 
-	types.RenderJSON(ctx, types.Success, nil, result)
+	types.RenderSuccess(ctx, result)
 }

bridge-history-api/internal/controller/controller.go

@@ -10,7 +10,12 @@ var (
 	// HistoryCtrler is controller instance
 	HistoryCtrler *HistoryController
 	// BatchCtrler is controller instance
-	BatchCtrler        *BatchController
+	BatchCtrler *BatchController
+	// HealthCheck the health check controller
+	HealthCheck *HealthCheckController
+	// Ready the ready controller
+	Ready *ReadyController
+
 	initControllerOnce sync.Once
 )
 
@@ -19,5 +24,7 @@ func InitController(db *gorm.DB) {
 	initControllerOnce.Do(func() {
 		HistoryCtrler = NewHistoryController(db)
 		BatchCtrler = NewBatchController(db)
+		HealthCheck = NewHealthCheckController(db)
+		Ready = NewReadyController()
 	})
 }

bridge-history-api/internal/controller/health_check.go

@@ -0,0 +1,30 @@
+package controller
+
+import (
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+
+	"bridge-history-api/internal/types"
+	"bridge-history-api/utils"
+)
+
+// HealthCheckController is health check API
+type HealthCheckController struct {
+	db *gorm.DB
+}
+
+// NewHealthCheckController returns an HealthCheckController instance
+func NewHealthCheckController(db *gorm.DB) *HealthCheckController {
+	return &HealthCheckController{
+		db: db,
+	}
+}
+
+// HealthCheck the api controller for coordinator health check
+func (a *HealthCheckController) HealthCheck(c *gin.Context) {
+	if _, err := utils.Ping(a.db); err != nil {
+		types.RenderFatal(c, err)
+		return
+	}
+	types.RenderSuccess(c, nil)
+}

bridge-history-api/internal/controller/history_controller.go

@@ -25,18 +25,18 @@ func NewHistoryController(db *gorm.DB) *HistoryController {
 func (c *HistoryController) GetAllClaimableTxsByAddr(ctx *gin.Context) {
 	var req types.QueryByAddressRequest
 	if err := ctx.ShouldBind(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, err)
 		return
 	}
 	offset := (req.Page - 1) * req.PageSize
 	limit := req.PageSize
 	txs, total, err := c.historyLogic.GetClaimableTxsByAddress(ctx, common.HexToAddress(req.Address), offset, limit)
 	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetClaimablesFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrGetClaimablesFailure, err)
 		return
 	}
 
-	types.RenderJSON(ctx, types.Success, nil, &types.ResultData{Result: txs, Total: total})
+	types.RenderSuccess(ctx, &types.ResultData{Result: txs, Total: total})
 }
 
 // GetAllTxsByAddr defines the http get method behavior
@@ -50,23 +50,23 @@ func (c *HistoryController) GetAllTxsByAddr(ctx *gin.Context) {
 	limit := req.PageSize
 	message, total, err := c.historyLogic.GetTxsByAddress(ctx, common.HexToAddress(req.Address), offset, limit)
 	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetTxsByAddrFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrGetTxsByAddrFailure, err)
 		return
 	}
-	types.RenderJSON(ctx, types.Success, nil, &types.ResultData{Result: message, Total: total})
+	types.RenderSuccess(ctx, &types.ResultData{Result: message, Total: total})
 }
 
 // PostQueryTxsByHash defines the http post method behavior
 func (c *HistoryController) PostQueryTxsByHash(ctx *gin.Context) {
 	var req types.QueryByHashRequest
 	if err := ctx.ShouldBindJSON(&req); err != nil {
-		types.RenderJSON(ctx, types.ErrParameterInvalidNo, err, nil)
+		types.RenderFailure(ctx, types.ErrParameterInvalidNo, err)
 		return
 	}
 	result, err := c.historyLogic.GetTxsByHashes(ctx, req.Txs)
 	if err != nil {
-		types.RenderJSON(ctx, types.ErrGetTxsByHashFailure, err, nil)
+		types.RenderFailure(ctx, types.ErrGetTxsByHashFailure, err)
 		return
 	}
-	types.RenderJSON(ctx, types.Success, nil, &types.ResultData{Result: result, Total: 0})
+	types.RenderSuccess(ctx, &types.ResultData{Result: result, Total: 0})
 }

bridge-history-api/internal/controller/ready.go

@@ -0,0 +1,21 @@
+package controller
+
+import (
+	"github.com/gin-gonic/gin"
+
+	"bridge-history-api/internal/types"
+)
+
+// ReadyController ready API
+type ReadyController struct {
+}
+
+// NewReadyController returns an ReadyController instance
+func NewReadyController() *ReadyController {
+	return &ReadyController{}
+}
+
+// Ready the api controller for coordinator ready
+func (r *ReadyController) Ready(c *gin.Context) {
+	types.RenderSuccess(c, nil)
+}

bridge-history-api/internal/route/route.go

@@ -25,4 +25,6 @@ func Route(router *gin.Engine, conf *config.Config) {
 	r.POST("/txsbyhashes", controller.HistoryCtrler.PostQueryTxsByHash)
 	r.GET("/claimable", controller.HistoryCtrler.GetAllClaimableTxsByAddr)
 	r.GET("/withdraw_root", controller.BatchCtrler.GetWithdrawRootByBatchIndex)
+	r.GET("/health", controller.HealthCheck.HealthCheck)
+	r.GET("/ready", controller.Ready.Ready)
 }

bridge-history-api/internal/types/history_types.go

@@ -10,6 +10,8 @@ import (
 const (
 	// Success shows OK.
 	Success = 0
+	// InternalServerError shows a fatal error in the server
+	InternalServerError = 500
 	// ErrParameterInvalidNo is invalid params
 	ErrParameterInvalidNo = 40001
 	// ErrGetClaimablesFailure is getting all claimables txs error
@@ -103,3 +105,28 @@ func RenderJSON(ctx *gin.Context, errCode int, err error, data interface{}) {
 	}
 	ctx.JSON(http.StatusOK, renderData)
 }
+
+// RenderSuccess renders success response with json
+func RenderSuccess(ctx *gin.Context, data interface{}) {
+	RenderJSON(ctx, Success, nil, data)
+}
+
+// RenderFailure renders failure response with json
+func RenderFailure(ctx *gin.Context, errCode int, err error) {
+	RenderJSON(ctx, errCode, err, nil)
+}
+
+// RenderFatal renders fatal response with json
+func RenderFatal(ctx *gin.Context, err error) {
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	renderData := Response{
+		ErrCode: InternalServerError,
+		ErrMsg:  errMsg,
+		Data:    nil,
+	}
+	ctx.Set("errcode", InternalServerError)
+	ctx.JSON(http.StatusInternalServerError, renderData)
+}

bridge-history-api/utils/database.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -67,18 +68,29 @@ func InitDB(config *config.DBConfig) (*gorm.DB, error) {
 	if err != nil {
 		return nil, err
 	}
-	sqlDB, err := db.DB()
-	if err != nil {
-		return nil, err
+
+	sqlDB, pingErr := Ping(db)
+	if pingErr != nil {
+		return nil, pingErr
 	}
 
 	sqlDB.SetMaxOpenConns(config.MaxOpenNum)
 	sqlDB.SetMaxIdleConns(config.MaxIdleNum)
 
+	return db, nil
+}
+
+// Ping check db status
+func Ping(db *gorm.DB) (*sql.DB, error) {
+	sqlDB, err := db.DB()
+	if err != nil {
+		return nil, err
+	}
+
 	if err = sqlDB.Ping(); err != nil {
 		return nil, err
 	}
-	return db, nil
+	return sqlDB, nil
 }
 
 // CloseDB close the db handler. notice the db handler only can close when then program exit.

common/database/db.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -63,17 +64,15 @@ func InitDB(config *Config) (*gorm.DB, error) {
 	if err != nil {
 		return nil, err
 	}
-	sqlDB, err := db.DB()
-	if err != nil {
-		return nil, err
+
+	sqlDB, pingErr := Ping(db)
+	if pingErr != nil {
+		return nil, pingErr
 	}
 
 	sqlDB.SetMaxOpenConns(config.MaxOpenNum)
 	sqlDB.SetMaxIdleConns(config.MaxIdleNum)
 
-	if err = sqlDB.Ping(); err != nil {
-		return nil, err
-	}
 	return db, nil
 }
 
@@ -88,3 +87,16 @@ func CloseDB(db *gorm.DB) error {
 	}
 	return nil
 }
+
+// Ping check db status
+func Ping(db *gorm.DB) (*sql.DB, error) {
+	sqlDB, err := db.DB()
+	if err != nil {
+		return nil, err
+	}
+
+	if err = sqlDB.Ping(); err != nil {
+		return nil, err
+	}
+	return sqlDB, nil
+}

common/database/db_test.go

@@ -11,6 +11,10 @@ import (
 	"github.com/mattn/go-colorable"
 	"github.com/mattn/go-isatty"
 	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/stretchr/testify/assert"
+
+	"scroll-tech/common/docker"
+	"scroll-tech/common/version"
 )
 
 func TestGormLogger(t *testing.T) {
@@ -33,3 +37,26 @@ func TestGormLogger(t *testing.T) {
 	gl.Info(context.Background(), "test %s warn:%v", "testInfo", errors.New("test info"))
 	gl.Trace(context.Background(), time.Now(), func() (string, int64) { return "test trace", 1 }, nil)
 }
+
+func TestDB(t *testing.T) {
+	version.Version = "v4.1.98-aaa-bbb-ccc"
+	base := docker.NewDockerApp()
+	base.RunDBImage(t)
+
+	dbCfg := &Config{
+		DSN:        base.DBConfig.DSN,
+		DriverName: base.DBConfig.DriverName,
+		MaxOpenNum: base.DBConfig.MaxOpenNum,
+		MaxIdleNum: base.DBConfig.MaxIdleNum,
+	}
+
+	var err error
+	db, err := InitDB(dbCfg)
+	assert.NoError(t, err)
+
+	sqlDB, err := Ping(db)
+	assert.NoError(t, err)
+	assert.NotNil(t, sqlDB)
+
+	assert.NoError(t, CloseDB(db))
+}

common/metrics/middleware.go → common/observability/middleware.go

@@ -1,10 +1,10 @@
-package metrics
+package observability
 
 import (
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"scroll-tech/common/metrics/ginmetrics"
+	"scroll-tech/common/observability/ginmetrics"
 )
 
 // Use register the gin metric

common/observability/probes.go

@@ -0,0 +1,35 @@
+package observability
+
+import (
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/types"
+)
+
+// ProbesController probe check controller
+type ProbesController struct {
+	db *gorm.DB
+}
+
+// NewProbesController returns an ProbesController instance
+func NewProbesController(db *gorm.DB) *ProbesController {
+	return &ProbesController{
+		db: db,
+	}
+}
+
+// HealthCheck the api controller for health check
+func (a *ProbesController) HealthCheck(c *gin.Context) {
+	if _, err := database.Ping(a.db); err != nil {
+		types.RenderFatal(c, err)
+		return
+	}
+	types.RenderSuccess(c, nil)
+}
+
+// Ready the api controller for ready check
+func (a *ProbesController) Ready(c *gin.Context) {
+	types.RenderSuccess(c, nil)
+}