From 248c4c9c690669784f3cfa77c0fb91afa5ac2dbf Mon Sep 17 00:00:00 2001 From: orenzhang Date: Tue, 6 Feb 2024 15:45:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E9=85=8D=E7=BD=AE&=E5=A2=9E=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E8=BF=9E=E6=8E=A5=E9=85=8D=E7=BD=AE&=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=95=B0=E6=8D=AE=E5=BA=93=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + go.mod | 5 +- go.sum | 2 + src/configs/db.go | 32 ++++++++----- src/configs/default.go | 27 +++++++---- src/configs/redis.go | 22 +++++---- src/db/mysql.go | 8 +++- src/db/redis.go | 9 ++-- src/middlewares/request.go | 24 +++------- src/middlewares/timout.go | 20 ++++++++ src/server/engine.go | 4 +- src/server/server.go | 45 ++++++++++++++++-- src/utils/log.go | 97 +++++++++++++++++++++++++++++++++----- 13 files changed, 220 insertions(+), 76 deletions(-) create mode 100644 src/middlewares/timout.go diff --git a/.gitignore b/.gitignore index 485dee6..75a1f53 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .idea +*.log diff --git a/go.mod b/go.mod index 0b9aff2..89a2222 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module github.com/OVINC-CN/DevTemplateGo go 1.20 require ( + github.com/gin-contrib/timeout v0.0.6 github.com/gin-gonic/gin v1.9.1 + github.com/go-playground/validator/v10 v10.17.0 github.com/google/uuid v1.6.0 github.com/redis/go-redis/v9 v9.4.0 github.com/sirupsen/logrus v1.9.3 + golang.org/x/crypto v0.18.0 gorm.io/driver/mysql v1.5.2 gorm.io/gorm v1.25.6 ) @@ -21,7 +24,6 @@ require ( github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.17.0 // indirect github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect @@ -36,7 +38,6 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index ec4f765..4a0455e 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-contrib/timeout v0.0.6 h1:hRx+DnzQvHAsM7T3SGgIPOOMmyM/Z+szE5IDqf91Mog= +github.com/gin-contrib/timeout v0.0.6/go.mod h1:a4ovm+qCGc+PIK3oF/vm6lC03SG+zZlh+qlDz+7vpnA= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= diff --git a/src/configs/db.go b/src/configs/db.go index 53bcf8d..53dbea7 100644 --- a/src/configs/db.go +++ b/src/configs/db.go @@ -1,19 +1,29 @@ package configs -import "github.com/OVINC-CN/DevTemplateGo/src/utils" +import ( + "github.com/OVINC-CN/DevTemplateGo/src/utils" + "strconv" + "time" +) type dbConfigModel struct { - Host string - Port string - User string - Password string - Name string + Host string + Port string + User string + Password string + Name string + MaxConnections int + ConnectionTimeOut time.Duration + SlowThreshold time.Duration } var DBConfig = dbConfigModel{ - Host: utils.GetEnv("DB_HOST", "127.0.0.1"), - Port: utils.GetEnv("DB_PORT", "3306"), - User: utils.GetEnv("DB_USER", ""), - Password: utils.GetEnv("DB_PASSWORD", ""), - Name: utils.GetEnv("DB_NAME", ""), + Host: utils.GetEnv("DB_HOST", "127.0.0.1"), + Port: utils.GetEnv("DB_PORT", "3306"), + User: utils.GetEnv("DB_USER", ""), + Password: utils.GetEnv("DB_PASSWORD", ""), + Name: utils.GetEnv("DB_NAME", ""), + MaxConnections: utils.StrToInt(utils.GetEnv("DB_MAX_CONNECTIONS", "10")), + ConnectionTimeOut: time.Duration(utils.StrToInt(utils.GetEnv("DB_CONNECTION_TIMEOUT", strconv.Itoa(60*60)))) * time.Second, + SlowThreshold: time.Duration(utils.StrToInt(utils.GetEnv("DB_SLOW_THRESHOLD", strconv.Itoa(100)))) * time.Millisecond, } diff --git a/src/configs/default.go b/src/configs/default.go index 4305902..df05e3b 100644 --- a/src/configs/default.go +++ b/src/configs/default.go @@ -4,20 +4,27 @@ import ( "github.com/OVINC-CN/DevTemplateGo/src/utils" "github.com/sirupsen/logrus" "os" + "time" ) type configModel struct { - Debug bool - LogLevel logrus.Level - Port string - AppCode string - AppSecret string + AppCode string + AppSecret string + Debug bool + LogLevel logrus.Level + Port string + RequestTimeout time.Duration + TLSCert string + TLSKey string } var Config = configModel{ - Debug: utils.StrToBool(utils.GetEnv("DEBUG", "false")), - LogLevel: logrus.InfoLevel, - Port: utils.GetEnv("PORT", ":8000"), - AppCode: os.Getenv("APP_CODE"), - AppSecret: os.Getenv("APP_SECRET"), + AppCode: os.Getenv("APP_CODE"), + AppSecret: os.Getenv("APP_SECRET"), + Debug: utils.StrToBool(utils.GetEnv("DEBUG", "false")), + LogLevel: logrus.InfoLevel, + Port: utils.GetEnv("PORT", ":8000"), + RequestTimeout: time.Duration(utils.StrToInt(utils.GetEnv("REQUEST_TIMEOUT", "10"))) * time.Second, + TLSCert: utils.GetEnv("TLS_CERT", ""), + TLSKey: utils.GetEnv("TLS_KEY", ""), } diff --git a/src/configs/redis.go b/src/configs/redis.go index 9533daf..f8d26a8 100644 --- a/src/configs/redis.go +++ b/src/configs/redis.go @@ -3,17 +3,19 @@ package configs import "github.com/OVINC-CN/DevTemplateGo/src/utils" type redisConfigModel struct { - Host string - Port string - Password string - DB int - Prefix string + Host string + Port string + Password string + DB int + Prefix string + MaxConnections int } var RedisConfig = redisConfigModel{ - Host: utils.GetEnv("REDIS_HOST", "127.0.0.1"), - Port: utils.GetEnv("REDIS_PORT", "6379"), - Password: utils.GetEnv("REDIS_PASSWORD", ""), - DB: utils.StrToInt(utils.GetEnv("REDIS_DB", "0")), - Prefix: utils.GetEnv("REDIS_PREFIX", ""), + Host: utils.GetEnv("REDIS_HOST", "127.0.0.1"), + Port: utils.GetEnv("REDIS_PORT", "6379"), + Password: utils.GetEnv("REDIS_PASSWORD", ""), + DB: utils.StrToInt(utils.GetEnv("REDIS_DB", "0")), + Prefix: utils.GetEnv("REDIS_PREFIX", ""), + MaxConnections: utils.StrToInt(utils.GetEnv("REDIS_MAX_CONNECTIONS", "10")), } diff --git a/src/db/mysql.go b/src/db/mysql.go index 6f90026..534fdb4 100644 --- a/src/db/mysql.go +++ b/src/db/mysql.go @@ -5,14 +5,18 @@ import ( "github.com/OVINC-CN/DevTemplateGo/src/utils" "gorm.io/driver/mysql" "gorm.io/gorm" + "time" ) var DB *gorm.DB -func InitDBConnection(host, port, user, password, name string) { +func InitDBConnection(host, port, user, password, name string, maxConnections int, connectionTimeout time.Duration, config *gorm.Config) { var err error dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4", user, password, host, port, name) - DB, err = gorm.Open(mysql.Open(dsn), &gorm.Config{}) + DB, err = gorm.Open(mysql.Open(dsn), config) + sqlDB, err := DB.DB() + sqlDB.SetMaxOpenConns(maxConnections) + sqlDB.SetConnMaxLifetime(connectionTimeout) if err != nil { utils.Logger.Errorf("[InitDBConnectionFailed] %s", err) } else { diff --git a/src/db/redis.go b/src/db/redis.go index f1d4332..1c10982 100644 --- a/src/db/redis.go +++ b/src/db/redis.go @@ -9,11 +9,12 @@ import ( var Redis *redis.Client -func InitRedisConnection(host, port, password string, db int) { +func InitRedisConnection(host, port, password string, db, maxConnections int) { Redis = redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%s", host, port), - Password: password, - DB: db, + Addr: fmt.Sprintf("%s:%s", host, port), + Password: password, + DB: db, + MaxActiveConns: maxConnections, }) status := Redis.Ping(context.Background()) utils.Logger.Infof("[InitRedisConnectionSuccess] %T %s", Redis, status) diff --git a/src/middlewares/request.go b/src/middlewares/request.go index 31e86db..b052bff 100644 --- a/src/middlewares/request.go +++ b/src/middlewares/request.go @@ -1,11 +1,9 @@ package middlewares import ( - "bytes" - "encoding/json" + "github.com/OVINC-CN/DevTemplateGo/src/services/account" "github.com/OVINC-CN/DevTemplateGo/src/utils" "github.com/gin-gonic/gin" - "io" "time" ) @@ -13,26 +11,16 @@ func RequestLogger() gin.HandlerFunc { return func(c *gin.Context) { // 初始化请求时间 t := time.Now() - // 提取请求体 - requestRawData, err := c.GetRawData() - c.Request.Body = io.NopCloser(bytes.NewReader(requestRawData)) - if err != nil { - utils.ContextWarningf(c, "[LoadRequestBodyFailed] %s", err) - } // 执行 c.Next() // 记录请求耗时 duration := time.Since(t).Milliseconds() - // 解析请求体 - var requestJsonData map[string]interface{} - if len(requestRawData) > 0 { - if err = json.Unmarshal(requestRawData, &requestJsonData); err != nil { - requestRawData = []byte("-") - } - } else { - requestRawData = []byte("-") + // 记录用户 + username := account.GetContextUser(c).Username + if username == "" { + username = "-" } // 记录请求日志 - utils.ContextInfof(c, "[RequestLog] (%dms) %s %s %s %d", duration, c.Request.Method, c.Request.URL, requestRawData, c.Writer.Status()) + utils.ContextInfof(c, "[RequestLog] %s %s %s %d %d", username, c.Request.Method, c.Request.URL, duration, c.Writer.Status()) } } diff --git a/src/middlewares/timout.go b/src/middlewares/timout.go new file mode 100644 index 0000000..0e85eb6 --- /dev/null +++ b/src/middlewares/timout.go @@ -0,0 +1,20 @@ +package middlewares + +import ( + "github.com/OVINC-CN/DevTemplateGo/src/configs" + "github.com/gin-contrib/timeout" + "github.com/gin-gonic/gin" + "net/http" +) + +func Timeout() gin.HandlerFunc { + return timeout.New( + timeout.WithTimeout(configs.Config.RequestTimeout), + timeout.WithHandler(func(c *gin.Context) { + c.Next() + }), + timeout.WithResponse(func(c *gin.Context) { + c.AbortWithStatusJSON(http.StatusGatewayTimeout, gin.H{"error": "timeout"}) + }), + ) +} diff --git a/src/server/engine.go b/src/server/engine.go index 8aaf8e0..7c42abd 100644 --- a/src/server/engine.go +++ b/src/server/engine.go @@ -17,13 +17,13 @@ func setupRouter() (engine *gin.Engine) { } engine = gin.New() engine.RedirectTrailingSlash = false - engine.Use(middlewares.InitLogger(), middlewares.RequestLogger(), middlewares.Authenticate()) + engine.Use(middlewares.InitLogger(), middlewares.RequestLogger(), middlewares.Timeout(), middlewares.Authenticate()) // 注册校验器 if v, ok := binding.Validator.Engine().(*validator.Validate); ok { _ = v.RegisterValidation("username", account.UsernameValidator) } // Home - homeGroup := engine.Group("/home/") + homeGroup := engine.Group("/") { homeGroup.GET("", home.Home) } diff --git a/src/server/server.go b/src/server/server.go index 4a9d4ed..3209bdb 100644 --- a/src/server/server.go +++ b/src/server/server.go @@ -5,19 +5,56 @@ import ( "github.com/OVINC-CN/DevTemplateGo/src/db" "github.com/OVINC-CN/DevTemplateGo/src/services/account" "github.com/OVINC-CN/DevTemplateGo/src/utils" + "gorm.io/gorm" + "runtime" ) func startServer() { // init log - utils.InitLogger(configs.Config.Debug, configs.Config.LogLevel) + utils.Logger = utils.InitLogger( + configs.Config.Debug, + configs.Config.LogLevel, + ) + utils.DbLogger = utils.InitDBLogger( + configs.Config.Debug, + configs.Config.LogLevel, + configs.DBConfig.SlowThreshold, + ) // init db - db.InitDBConnection(configs.DBConfig.Host, configs.DBConfig.Port, configs.DBConfig.User, configs.DBConfig.Password, configs.DBConfig.Name) + db.InitDBConnection( + configs.DBConfig.Host, + configs.DBConfig.Port, + configs.DBConfig.User, + configs.DBConfig.Password, + configs.DBConfig.Name, + configs.DBConfig.MaxConnections, + configs.DBConfig.ConnectionTimeOut, + &gorm.Config{ + Logger: utils.DbLogger, + }, + ) migrate() // init redis - db.InitRedisConnection(configs.RedisConfig.Host, configs.RedisConfig.Port, configs.RedisConfig.Password, configs.RedisConfig.DB) + db.InitRedisConnection( + configs.RedisConfig.Host, + configs.RedisConfig.Port, + configs.RedisConfig.Password, + configs.RedisConfig.DB, + configs.RedisConfig.MaxConnections, + ) + // init cpu + threads := runtime.NumCPU() + runtime.GOMAXPROCS(threads) + utils.Logger.Infof("[InitCPUSuccess] Runs on %d CPUs", threads) // init gin engine := setupRouter() - if err := engine.Run(configs.Config.Port); err != nil { + var err error + if configs.Config.TLSCert != "" { + err = engine.RunTLS(configs.Config.Port, configs.Config.TLSCert, configs.Config.TLSKey) + } else { + err = engine.Run(configs.Config.Port) + } + if err != nil { utils.Logger.Infof("[ServerStartFailed] %s", err) panic(err) } diff --git a/src/utils/log.go b/src/utils/log.go index 75d3b85..89d075d 100644 --- a/src/utils/log.go +++ b/src/utils/log.go @@ -1,39 +1,50 @@ package utils import ( + "context" + "fmt" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + gormLogger "gorm.io/gorm/logger" + "gorm.io/gorm/utils" "os" "time" ) var ( - Logger *logrus.Entry + Logger *logrus.Entry + DbLogger *DBLoggerType ) -func InitLogger(debug bool, level logrus.Level) { +func InitLogger(debug bool, level logrus.Level) *logrus.Entry { rawLogger := logrus.New() rawLogger.SetLevel(level) - rawLogger.SetFormatter(&logrus.JSONFormatter{ - TimestampFormat: time.RFC3339, - DisableTimestamp: false, - DisableHTMLEscape: true, - PrettyPrint: debug, - FieldMap: logrus.FieldMap{ - logrus.FieldKeyMsg: "message", - }, - }) if debug { + rawLogger.SetFormatter(&logrus.TextFormatter{ + DisableQuote: false, + DisableTimestamp: false, + TimestampFormat: time.RFC3339, + }) rawLogger.SetOutput(os.Stdout) } else { + rawLogger.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: time.RFC3339, + DisableTimestamp: false, + DisableHTMLEscape: true, + PrettyPrint: debug, + FieldMap: logrus.FieldMap{ + logrus.FieldKeyMsg: "message", + }, + }) file, err := os.OpenFile("server.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { panic(err) } rawLogger.SetOutput(file) } - Logger = rawLogger.WithFields(logrus.Fields{"request_id": ""}) - Logger.Infof("[InitLoggerSuccess] %T", Logger) + logger := rawLogger.WithFields(logrus.Fields{"request_id": ""}) + logger.Infof("[InitLoggerSuccess] %T", Logger) + return logger } func getContextLogger(c *gin.Context) (logger *logrus.Entry, exists bool) { @@ -60,3 +71,63 @@ func ContextWarningf(c *gin.Context, format string, args ...interface{}) { logger, _ := getContextLogger(c) logger.Warningf(format, args...) } + +type DBLoggerType struct { + Logger *logrus.Entry + infoStr, warnStr, errStr string + traceStr, traceErrStr, traceWarnStr string + slowThreshold time.Duration +} + +func (l *DBLoggerType) LogMode(level gormLogger.LogLevel) gormLogger.Interface { + newLogger := *l + return &newLogger +} + +func (l *DBLoggerType) Info(ctx context.Context, msg string, data ...interface{}) { + l.Logger.Infof(l.infoStr+msg, append([]interface{}{utils.FileWithLineNum()}, data...)...) +} + +func (l *DBLoggerType) Warn(ctx context.Context, msg string, data ...interface{}) { + l.Logger.Warningf(l.warnStr+msg, append([]interface{}{utils.FileWithLineNum()}, data...)...) +} + +func (l *DBLoggerType) Error(ctx context.Context, msg string, data ...interface{}) { + l.Logger.Errorf(l.errStr+msg, append([]interface{}{utils.FileWithLineNum()}, data...)...) +} + +func (l *DBLoggerType) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) { + elapsed := time.Since(begin) + switch { + case elapsed > l.slowThreshold && l.slowThreshold != 0: + sql, rows := fc() + slowLog := fmt.Sprintf("SLOW SQL >= %v", l.slowThreshold) + if rows == -1 { + l.Logger.Warningf(l.traceWarnStr, utils.FileWithLineNum(), slowLog, float64(elapsed.Nanoseconds())/1e6, "-", sql) + } else { + l.Logger.Warningf(l.traceWarnStr, utils.FileWithLineNum(), slowLog, float64(elapsed.Nanoseconds())/1e6, rows, sql) + } + case l.Logger.Level == logrus.InfoLevel: + sql, rows := fc() + if rows == -1 { + l.Logger.Infof(l.traceStr, utils.FileWithLineNum(), float64(elapsed.Nanoseconds())/1e6, "-", sql) + } else { + l.Logger.Infof(l.traceStr, utils.FileWithLineNum(), float64(elapsed.Nanoseconds())/1e6, rows, sql) + } + } +} + +func InitDBLogger(debug bool, level logrus.Level, slowThreshold time.Duration) *DBLoggerType { + logger := InitLogger(debug, level) + dbLogger := &DBLoggerType{ + Logger: logger, + slowThreshold: slowThreshold, + infoStr: "[Database] %s", + warnStr: "[Database] %s", + errStr: "[Database] %s", + traceStr: "[DatabaseTrace] %s\n[%.3fms] [rows:%v] %s", + traceWarnStr: "[DatabaseTrace] %s %s\n[%.3fms] [rows:%v] %s", + traceErrStr: "[DatabaseTrace] %s %s\n[%.3fms] [rows:%v] %s", + } + return dbLogger +}