diff --git a/dbm-services/common/db-resource/go.mod b/dbm-services/common/db-resource/go.mod index a498c72f00..9dd329ca86 100644 --- a/dbm-services/common/db-resource/go.mod +++ b/dbm-services/common/db-resource/go.mod @@ -68,4 +68,4 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect -) +) \ No newline at end of file diff --git a/dbm-services/common/db-resource/go.sum b/dbm-services/common/db-resource/go.sum index 970b21f6c8..32bf6268db 100644 --- a/dbm-services/common/db-resource/go.sum +++ b/dbm-services/common/db-resource/go.sum @@ -36,8 +36,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= @@ -66,12 +70,24 @@ github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17 github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dhui/dktest v0.4.3 h1:wquqUxAFdcUgabAVLvSCOKOlag5cIZuaOjYIBOWdsR0= +github.com/dhui/dktest v0.4.3/go.mod h1:zNK8IwktWzQRm6I/l2Wjp7MakiyaFWv4G1hjmodmMTs= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4= +github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -91,8 +107,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -115,6 +131,10 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.18.1 h1:JML/k+t4tpHCpQTCAD62Nu43NUFzHY4CV3uAuvHGC+Y= +github.com/golang-migrate/migrate/v4 v4.18.1/go.mod h1:HAX6m3sQgcdO81tdjn5exv20+3Kb13cmGli1hrD6hks= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -172,11 +192,16 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -209,6 +234,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -220,21 +247,32 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -245,8 +283,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -272,8 +310,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cbs v1.0.750 h1:ZXvq/25p4Gs3zCY1dGFUq5g8JaVOPVVEBEcMCwAVpYY= @@ -298,14 +337,18 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.46.1 h1:mMv2jG58h6ZI5t5S9QCVGdzCmAsTakMa3oxVgpSD44g= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.46.1/go.mod h1:oqRuNKG0upTaDPbLVCG8AD0G2ETrfDtmh7jViy7ox6M= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/contrib/propagators/b3 v1.21.1 h1:WPYiUgmw3+b7b3sQ1bFBFAf0q+Di9dvNc3AtYfnT4RQ= go.opentelemetry.io/contrib/propagators/b3 v1.21.1/go.mod h1:EmzokPoSqsYMBVK4nRnhsfm5mbn8J1eDuz/U1UaQaWg= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -317,8 +360,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -329,8 +372,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -386,8 +429,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -447,8 +490,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -458,8 +501,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -604,8 +647,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/dbm-services/common/go-pubpkg/go.mod b/dbm-services/common/go-pubpkg/go.mod index 7c148a5bfe..c7c090b027 100644 --- a/dbm-services/common/go-pubpkg/go.mod +++ b/dbm-services/common/go-pubpkg/go.mod @@ -31,6 +31,7 @@ require ( go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + gopkg.in/ini.v1 v1.67.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -92,6 +93,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/dbm-services/common/go-pubpkg/mysqlcomm/mycnf.go b/dbm-services/common/go-pubpkg/mysqlcomm/mycnf.go new file mode 100644 index 0000000000..96310d7b60 --- /dev/null +++ b/dbm-services/common/go-pubpkg/mysqlcomm/mycnf.go @@ -0,0 +1,499 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package mysqlcomm + +import ( + "fmt" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "gopkg.in/ini.v1" + + "dbm-services/common/go-pubpkg/cmutil" +) + +const ( + // MysqldSec TODO + MysqldSec = "mysqld" + DefaultMyCnfName = "/etc/my.cnf" +) + +// CnfFile TODO +type CnfFile struct { + FileName string + Cfg *ini.File + mu *sync.Mutex +} + +// CnfUint TODO +type CnfUint struct { + KvMap map[string]string + // 可重复的key + ShadowKvMap map[string]string + // skip_symbolic_links单key的配置 + BoolKey []string +} + +// MycnfIniObject TODO +type MycnfIniObject struct { + Section map[string]*CnfUint +} + +var iniLoadOption = ini.LoadOptions{ + PreserveSurroundedQuote: true, + IgnoreInlineComment: true, + AllowBooleanKeys: true, + AllowShadows: true, +} + +// LoadMyCnfForFile 读取一个已经存在的配置文件，将配置文件的内容解析,用于程序读取、修改my.cnf +// +// @receiver mycnf +// @return *CnfFile +// @return error +func LoadMyCnfForFile(mycnf string) (*CnfFile, error) { + if err := cmutil.FileExistsErr(mycnf); err != nil { + return nil, err + } + cfg, err := ini.LoadSources(iniLoadOption, mycnf) + if err != nil { + return nil, err + } + return &CnfFile{ + FileName: mycnf, + mu: &sync.Mutex{}, + Cfg: cfg, + }, nil +} + +func newMyCnfUint() *CnfUint { + return &CnfUint{ + KvMap: make(map[string]string), + ShadowKvMap: make(map[string]string), + BoolKey: make([]string, 0), + } +} + +// Load load m.FileName to CnfOj +func (m *CnfFile) Load() error { + if obj, err := LoadMyCnfForFile(m.FileName); err != nil { + return err + } else { + m.Cfg = obj.Cfg + if m.mu == nil { + m.mu = &sync.Mutex{} + } + } + return nil +} + +// GetMySQLDataDir 从my.cnf 获取datadir +// +// @receiver m +// @return datadir +// @return err +// +// e.g: datadir=/data1/mysqldata/20000/data 返回 /data1/mysqldata/20000 +// datadir=/data/mysqldata/data 返回 /data/mysqldata/ +func (m *CnfFile) GetMySQLDataDir() (string, error) { + var datadir string + if m.Cfg.Section(MysqldSec).HasKey("datadir") { + datadir = m.Cfg.Section(MysqldSec).Key("datadir").String() + } else { + return "", fmt.Errorf("在配置中没找到datadir的配置项") + } + // datadirName := filepath.Base(datadir) // "data" + datadirPath := filepath.Dir(datadir) // "/data1/mysqldata/20000" or "/data/mysqldata" + if _, err := strconv.Atoi(filepath.Base(datadirPath)); err != nil { // "/data/mysqldata" + return datadir, nil // "/data/mysqldata/data" + } else { + return filepath.Dir(datadir), nil // "/data/mysqldata/20000/data" + } +} + +// GetMySQLLogDir 从配置中获取mysql logdir +// +// @receiver m +// @return logdir +// @return err +func (m *CnfFile) GetMySQLLogDir() (logdir string, err error) { + // 先从 log_bin 配置项获取logdir + // 但是可能存在历史的实例并没有开始binlog + // log_bin = ON + // log_bin_basename = /data/mysqllog/20000/binlog/binlog20000 + // log_bin_index = /data/mysqllog/20000/binlog/binlog20000.index + // 或者 log_bin = /data/mysqllog/20000/binlog/binlog20000.bin + // 或者 slow_query_log_file = /data/mysqllog/20000/slow-query.log + keys := []string{"log_bin", "log_bin_basename", "slow_query_log_file"} + + for _, k := range keys { + if val, err := m.GetMySQLCnfByKey(MysqldSec, k); err == nil { + if filepath.IsAbs(val) { + return val, nil + } + } + } + return "", fmt.Errorf("在配置中没找到 log_bin 的配置项") +} + +// GetBinLogDir 获取 binlog dir +// 这里只从 my.cnf 获取，有可能没有设置选项，外部需要考虑再次从 global variables 获取 +// 返回 binlog 目录和 binlog 文件名前缀 +func (m *CnfFile) GetBinLogDir() (binlogDir, namePrefix string, err error) { + // log_bin = ON + // log_bin_basename = /data/mysqllog/20000/binlog/binlog20000 // binlog 在没有指定路径的情况下，默认存放在 datadir + // log_bin_index = /data/mysqllog/20000/binlog/binlog20000.index + // 或者 log_bin = /data/mysqllog/20000/binlog/binlog20000.bin + keys := []string{"log_bin", "log_bin_basename"} + for _, k := range keys { + if val, err := m.GetMySQLCnfByKey(MysqldSec, k); err == nil { + if filepath.IsAbs(val) { + if binlogDir, namePrefix, err = m.ParseLogBinBasename(val); err == nil { + return binlogDir, namePrefix, nil + } + } + } + } + return "", "", fmt.Errorf("binlog dir not found or parse failed") +} + +// ParseLogBinBasename TODO +func (m *CnfFile) ParseLogBinBasename(val string) (binlogDir, namePrefix string, err error) { + binlogDir, namePrefix = path.Split(val) + if cmutil.IsDirectory(binlogDir) && !cmutil.IsDirectory(val) { + if strings.Contains(namePrefix, ".") { + binlogFilename := strings.Split(namePrefix, ".") + namePrefix = binlogFilename[0] + } + return binlogDir, namePrefix, nil + } else { + return "", "", errors.Errorf("expect %s is a dir and %s is not dir", binlogDir, val) + } +} + +// GetRelayLogDir TODO +func (m *CnfFile) GetRelayLogDir() (string, error) { + // relay-log = /data1/mysqldata/20000/relay-log/relay-log.bin + // 或者 relay_log_basename = /data1/mysqldata/20000/relay-log/relay-bin + keys := []string{"relay_log", "relay_log_basename"} + for _, k := range keys { + if val, err := m.GetMySQLCnfByKey(MysqldSec, k); err == nil { + if filepath.IsAbs(val) { // 必须是绝对路径 + return val, nil + } + } + } + return "", fmt.Errorf("在配置中没找到 relay 的配置项") +} + +// GetMySQLSocket 从my.cnf中获取socket value +// +// @receiver m +// @return socket +// @return err +func (m *CnfFile) GetMySQLSocket() (socket string, err error) { + if m.Cfg.Section(MysqldSec).HasKey("socket") { + return m.Cfg.Section(MysqldSec).Key("socket").String(), nil + } + return "", fmt.Errorf("在配置中没找到socket的配置项") +} + +// GetMySQLCnfByKey 从 my.cnf 获取 key 对应的 value +// 允许替换 _, - +// 如果 section 为空，会尝试从 key 中以 . 切分 section +func (m *CnfFile) GetMySQLCnfByKey(section, key string) (string, error) { + if section == "" { + sk := GetSectionFromKey(key, false) + key = sk.Key + section = sk.Section + } + key = m.GetKeyFromFile(section, key) + if m.Cfg.Section(section).HasKey(key) { + } else { + return "", fmt.Errorf("在配置中没找到 %s 的配置项", key) + } + return m.Cfg.Section(section).Key(key).String(), nil +} + +// GetMyCnfByKeyWithDefault TODO +func (m *CnfFile) GetMyCnfByKeyWithDefault(section, key string, valueDefault string) string { + if val, err := m.GetMySQLCnfByKey(section, key); err != nil { + return valueDefault + } else { + return val + } +} + +// SaveMySQLConfig2Object 将 my.cnf 变成 key map +func (m *CnfFile) SaveMySQLConfig2Object() MycnfIniObject { + var object MycnfIniObject + object.Section = make(map[string]*CnfUint) + for _, section := range m.Cfg.SectionStrings() { + object.Section[section] = newMyCnfUint() + for _, keyName := range m.Cfg.Section(section).KeyStrings() { + if kv, err := m.Cfg.Section(section).GetKey(keyName); err == nil { + object.Section[section].KvMap[keyName] = kv.Value() + } + } + } + return object +} + +// FastSaveChange 快速修改一个配置项，并持久化到文件 +func (m *CnfFile) FastSaveChange(port int, section, key, value string) (err error) { + m.mu.Lock() + defer m.mu.Unlock() + // 假如删除一个不存在的Key,不会抛出异常 + m.Cfg.Section(section).DeleteKey(key) + if _, err = m.Cfg.Section(section).NewKey(key, value); err != nil { + return + } + err = m.Cfg.SaveTo(fmt.Sprintf("my.cnf.%d", port)) + return +} + +// SafeSaveFile 全量持久化配置文件 +func (m *CnfFile) SafeSaveFile(isProxy bool) (err error) { + m.mu.Lock() + defer m.mu.Unlock() + nf, err := m.sortAllkeys(isProxy) + if err != nil { + return err + } + err = nf.Cfg.SaveTo(m.FileName) + return +} + +// sortAllkeys 对写入的key进行排序 +// +// @receiver m +// @return *CnfFile +// @return error +func (m *CnfFile) sortAllkeys(isProxy bool) (*CnfFile, error) { + f := CnfFile{ + Cfg: ini.Empty(iniLoadOption), + mu: &sync.Mutex{}, + } + for _, sec := range m.Cfg.Sections() { + secName := sec.Name() + if _, err := f.Cfg.NewSection(secName); err != nil { + return nil, err + } + keys := m.Cfg.Section(secName).KeyStrings() + sort.Strings(keys) + for _, key := range keys { + if m.isShadowKey(key) { + for _, val := range m.Cfg.Section(secName).Key(key).ValueWithShadows() { + f.RenderSection(secName, key, val, isProxy) + } + } + f.RenderSection(secName, key, m.Cfg.Section(secName).Key(key).Value(), isProxy) + } + } + return &f, nil +} + +// isShadowKey TODO +// 表示下面的key,可以在配置文件重复出现 +func (m *CnfFile) isShadowKey(key string) bool { + key = strings.ReplaceAll(key, "-", "_") + sk := []string{ + "replicate_do_db", "replicate_ignore_db", "replicate_do_table", "replicate_wild_do_table", + "replicate_ignore_table", "replicate_wild_ignore_table", + } + return cmutil.HasElem(key, sk) +} + +// GetInitDirItemTpl TODO +func (m *CnfFile) GetInitDirItemTpl(initDirs map[string]string) (err error) { + mysqld, err := m.Cfg.GetSection(MysqldSec) + if err != nil { + return + } + for key := range initDirs { + initDirs[key] = mysqld.Key(key).String() + } + return +} + +// RenderSection 替换渲染配置,proxy keepalive=true 不能和mysql的bool一样进行渲染 +// +// @receiver f +// @receiver sectionName +// @receiver key +// @receiver val +// @receiver isProxy +// @return err +func (m *CnfFile) RenderSection(sectionName, key, val string, isProxy bool) (err error) { + if m.isShadowKey(key) { + for _, shadowv := range strings.Split(val, ",") { + if _, err := m.Cfg.Section(sectionName).NewKey(key, shadowv); err != nil { + return err + } + // fmt.Println(",", "M") + } + return nil + } + // proxy.cnf 需要渲染 boolkey + // my.cnf 如果是空值，当做boolkey处理 + if !isProxy { + if strings.TrimSpace(val) == "true" { + if _, err = m.Cfg.Section(sectionName).NewBooleanKey(key); err != nil { + return err + } + return nil + } + } + // 如果是不是空值，当做boolkey处理 + if _, err = m.Cfg.Section(sectionName).NewKey(key, val); err != nil { + return err + } + return nil +} + +// ReplaceKeyName 存在oldkey，则替换为newkey; 不存在oldkey，则作任何处理 +// 比如用在把 default_charset_server 替换成 default-charset-server 的场景 +func (m *CnfFile) ReplaceKeyName(section string, oldKey string, newKey string) { + m.mu.Lock() + defer m.mu.Unlock() + + sel := m.Cfg.Section(section) + if sel.HasKey(oldKey) { + k, _ := sel.GetKey(oldKey) + sel.NewKey(newKey, k.Value()) + sel.DeleteKey(oldKey) + } +} + +// ReplaceMoreKv TODO +func (m *CnfFile) ReplaceMoreKv(pairs map[string]CnfUint) error { + if len(pairs) <= 0 { + return nil + } + for section, mu := range pairs { + for k, v := range mu.KvMap { + m.ReplaceValue(section, k, false, v) + } + } + return nil +} + +// GetKeyFromFile godoc +// my.cnf 里面允许 _, - 两种分隔符的变量，获取或者替换时，需要两种都尝试获取 +func (m *CnfFile) GetKeyFromFile(section string, key string) string { + if !m.Cfg.Section(section).HasKey(key) { + oldKey := key + key = strings.ReplaceAll(key, "_", "-") + if !m.Cfg.Section(section).HasKey(key) { + key = strings.ReplaceAll(key, "-", "_") + if !m.Cfg.Section(section).HasKey(key) { + key = oldKey // 始终没找到 key, 恢复输入值 + } + } + } + return key +} + +// ReplaceValue kv不存在则写入，k存在则更新 +// 会判断是否是 shadowKey +// 如果 key 不存在会尝试替换 _ 成 - +func (m *CnfFile) ReplaceValue(section string, key string, isBool bool, value string) { + m.mu.Lock() + defer m.mu.Unlock() + key = m.GetKeyFromFile(section, key) + if !m.isShadowKey(key) { + m.Cfg.Section(section).DeleteKey(key) + } + if isBool { + m.Cfg.Section(section).NewBooleanKey(key) + return + } + m.Cfg.Section(section).NewKey(key, value) +} + +// UpdateKeyValue 修改一个配置项，会判断是否是 shadowKey +func (m *CnfFile) UpdateKeyValue(section, key, value string) (err error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.isShadowKey(key) { + // 如果这些key 是可重复的key + err = m.Cfg.Section(section).Key(key).AddShadow(value) + return errors.Wrap(err, "update my.cnf KeyValue") + } + m.Cfg.Section(section).Key(key).SetValue(value) + return nil +} + +// GetMyCnfFileName 获取默认 my.cnf 的路径，不检查是否存在 +func GetMyCnfFileName(port int) string { + return fmt.Sprintf("%s.%d", DefaultMyCnfName, port) +} + +// ReplaceValuesToFile 文本替换 my.cnf 里面的 value，如果 key 不存在则插入 [mysqld] 后面 +func (m *CnfFile) ReplaceValuesToFile(newItems map[string]string) error { + if err := cmutil.OSCopyFile(m.FileName, fmt.Sprintf("%s.bak%s", + m.FileName, time.Now().Format("20060102150405"))); err != nil { + //logger.Warn("backup file %s failed, ignore: %s", m.FileName, err.Error()) + } + for k, v := range newItems { + if err := m.UpdateKeyValue(MysqldSec, k, v); err != nil { + return err + } + } + return m.SafeSaveFile(false) +} + +// GetMysqldKeyValue 增加基础方法，获取myconf上面的某个配置参数值，用于做前置校验的对比 +func (m *CnfFile) GetMysqldKeyValue(keyName string) (value string, err error) { + mysqld, err := m.Cfg.GetSection(MysqldSec) + if err != nil { + return "", err + } + return mysqld.Key(keyName).String(), nil +} + +// CnfKey my.cnf key格式 +type CnfKey struct { + Section string + Key string + IsBool bool + Separator string +} + +// GetSectionFromKey 从 . 分隔符里分离 section, key +// replace 控制是否将 - 替换为 _ (set global 用) +func GetSectionFromKey(key string, replace bool) *CnfKey { + sk := &CnfKey{Separator: "."} + ss := strings.Split(key, sk.Separator) + if len(ss) == 2 { + sk.Section = ss[0] + sk.Key = ss[1] + } else { + sk.Section = "" + sk.Key = key + } + if replace { + sk.Key = strings.ReplaceAll(sk.Key, "-", "_") + } + return sk +} + +// MycnfItemsMap mysqld变量映射到 my.cnf 中的配置名 +var MycnfItemsMap = map[string]string{ + "time_zone": "default-time-zone", + "character_set_system": "character_set_server", +} diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go index 2a31ff425e..fe2879eb36 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/jinzhu/copier" @@ -123,6 +124,7 @@ func (m *DBLoader) chooseDBBackupLoader() error { MyloaderOpt: myloaderOpt, } } else if m.backupType == cst.BackupTypePhysical { + // include rocksdb, tokudb m.dbLoader = &dbbackup_loader.PhysicalLoader{ LoaderUtil: m.dbLoaderUtil, Xtrabackup: &dbbackup_loader.Xtrabackup{ @@ -130,6 +132,7 @@ func (m *DBLoader) chooseDBBackupLoader() error { SrcBackupHost: m.dbLoaderUtil.IndexObj.BackupHost, QpressTool: m.Tools.MustGet(tools.ToolQPress), LoaderDir: m.targetDir, + StorageType: strings.ToLower(m.indexObj.StorageEngine), }, } } else { @@ -221,7 +224,7 @@ func (m *DBLoader) initDirs(removeOld bool) error { } } - m.taskDir = fmt.Sprintf("%s/doDr_%s/%d", m.WorkDir, m.WorkID, m.TgtInstance.Port) + m.taskDir = filepath.Join(m.WorkDir, fmt.Sprintf("doDr_%s/%d", m.WorkID, m.TgtInstance.Port)) if err := osutil.CheckAndMkdir("", m.taskDir); err != nil { return err } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/tokudb_recover.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/tokudb_recover.go new file mode 100644 index 0000000000..21e1cc0f7a --- /dev/null +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/tokudb_recover.go @@ -0,0 +1,18 @@ +package dbbackup_loader + +type PhysicalRecover interface { + PreRun() error + PostRun() error +} + +type TokudbRecover struct { + *Xtrabackup +} + +func (x *TokudbRecover) PreRun() error { + return x.PreRun() +} + +func (x *TokudbRecover) PostRun() error { + return x.PostRun() +} diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go similarity index 93% rename from dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup.go rename to dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go index f2e29f2cab..a3a5703c7a 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go @@ -29,6 +29,8 @@ type Xtrabackup struct { dbWorker *native.DbWorker // TgtInstance // 在 PreRun 时初始化，本地实例的配置文件 myCnf *util.CnfFile + + StorageType string `json:"-"` } // PreRun 以下所有步骤必须可重试 @@ -36,7 +38,6 @@ type Xtrabackup struct { // replace my.cnf func (x *Xtrabackup) PreRun() error { logger.Info("run xtrabackup preRun") - // 关闭本地mysql inst := x.TgtInstance @@ -97,9 +98,13 @@ func (x *Xtrabackup) PostRun() (err error) { serverVersion, err := x.dbWorker.SelectVersion() if err != nil { - //return errors.Wrapf(err, "get mysql version") - logger.Warn("get version failed: %s. set it to 5.7.20", err.Error()) - serverVersion = "5.7.20" // fake + if x.StorageType == "tokudb" { + logger.Warn("get version failed: %s. set it to 5.6.20 for tokudb", err.Error()) + serverVersion = "5.6.20" // fake + } else { + logger.Warn("get version failed: %s. set it to 5.7.20", err.Error()) + serverVersion = "5.7.20" // fake + } } logger.Info("repair user 'ADMIN' host and password") // 物理备份，ADMIN密码与 backup instance(cluster?) 相同，修复成 @@ -160,10 +165,15 @@ func (x *Xtrabackup) cleanXtraEnv() error { "datadir", "innodb_log_group_home_dir", "innodb_data_home_dir", + "tokudb_log_dir", + "tokudb_data_dir", "relay-log", "log_bin", "tmpdir", } + if x.StorageType == "tokudb" { + dirs = []string{"tokudb_log_dir", "tokudb_data_dir", "tmpdir"} + } return x.CleanEnv(dirs) } @@ -172,12 +182,13 @@ func (x *Xtrabackup) cleanXtraEnv() error { // mysql 8.0.30 之后 redo_log 变成 innodb_redo_log_capacity 来控制 func (x *Xtrabackup) doReplaceCnf() error { items := []string{ + "lower_case_table_names", "innodb_data_file_path", "innodb_log_files_in_group", "innodb_log_file_size", "innodb_page_size", "tokudb_cache_size", - "lower_case_table_names", + "rocksdb_block_cache_size", // mysql 8.0 xtrabackup "innodb_checksum_algorithm", @@ -204,6 +215,9 @@ func (x *Xtrabackup) changeDirOwner() error { "tmpdir", "log_bin", "slow_query_log_file", + + "tokudb_log_dir", + "tokudb_data_dir", } return x.ChangeDirOwner(dirs) } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_repaire.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go similarity index 98% rename from dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_repaire.go rename to dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go index b964ecfbd6..33e4fd013b 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_repaire.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go @@ -244,8 +244,8 @@ func (x *Xtrabackup) CleanEnv(dirs []string) error { } reg := regexp.MustCompile(cst.RelayLogFileMatch) if result := reg.FindStringSubmatch(val); len(result) == 2 { - relaylogdir := result[1] - dirArray = append(dirArray, "rm -rf "+relaylogdir+"/*") + relayLogDir := result[1] + dirArray = append(dirArray, "rm -rf "+relayLogDir+"/*") } case "log_bin", "log-bin": val, err := x.myCnf.GetMySQLLogDir() @@ -254,9 +254,9 @@ func (x *Xtrabackup) CleanEnv(dirs []string) error { } reg := regexp.MustCompile(cst.BinLogFileMatch) if result := reg.FindStringSubmatch(val); len(result) == 2 { - binlogdir := result[1] + binlogDir := result[1] // TODO 所有 rm -rf 的地方都应该要检查是否可能 rm -rf / binlog.xxx 这种误删可能 - dirArray = append(dirArray, "rm -rf "+binlogdir+"/*") + dirArray = append(dirArray, "rm -rf "+binlogDir+"/*") } case "slow_query_log_file", "slow-query-log-file": if val := x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, "slow_query_log_file", ""); val != "" { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_migrateold.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_migrateold.go index 83bee94fdd..f10c2f8fda 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_migrateold.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_migrateold.go @@ -125,6 +125,7 @@ func migrateOld(cmd *cobra.Command, args []string) (errs error) { if indexFilePath, indexContent, err := backupexe.MigrateInstanceBackupInfo(infoFilePath, &cnf); err != nil { //errMsg := fmt.Sprintf("failed migrate backup info port %d\n: %s", cnf.Public.MysqlPort, err.Error()) //errs = errors2.Join(errs, errors.New(errMsg)) + fmt.Println("xxxx err", err.Error()) continue } else { fmt.Println("migrate infoFile", infoFilePath) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/loader.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/loader.go index 418cd9165a..447c142ae5 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/loader.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/loader.go @@ -46,10 +46,12 @@ type LogicalLoadMysqldump struct { } // PhysicalLoad the config of physical loading +// works for innodb, rocksdb, tokudb type PhysicalLoad struct { MysqlLoadDir string `ini:"MysqlLoadDir" validate:"required"` IndexFilePath string `ini:"IndexFilePath" validate:"required,file"` + // DefaultsFile the target instance's my.cnf, not from backup files DefaultsFile string `ini:"DefaultsFile" validate:"required"` Threads int `ini:"Threads"` CopyBack bool `ini:"CopyBack"` // use copy-back or move-back diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go index b1f1497796..f9d2018a45 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go @@ -139,7 +139,7 @@ func (c *Public) TargetName() string { if c.targetName == "" { currentTime := time.Now().Format("20060102150405") c.targetName = fmt.Sprintf("%d_%d_%s_%d_%s_%s", - c.BkBizId, c.ClusterId, c.MysqlHost, c.MysqlPort, currentTime, c.BackupType) + c.BkBizId, c.ClusterId, c.MysqlHost, c.MysqlPort, currentTime, c.BackupType) // rocksdb/tokudb logger.Log.Info("generate target name: ", c.targetName) } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go index ce99034b43..cc94527926 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst/const.go @@ -18,6 +18,7 @@ const ( const ( StorageEngineRocksdb = "rocksdb" + StorageEngineTokudb = "tokudb" ) // backup role: dbm-services/mysql/db-tools/dbactuator/pkg/core/cst/mysql.go @@ -121,4 +122,5 @@ const ( ToolXtrabackup = "xtrabackup" ToolTmysqldump = "tmysqldump" ToolMyrocksHotbackup = "myrocks_hotbackup" + ToolTokudbBackup = "tokudb_back.pl" ) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go index 4460e6e7cd..d2bba97cf4 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go @@ -71,6 +71,10 @@ func BuildDumper(cnf *config.BackupConfig, storageEngine string) (dumper Dumper, dumper = &PhysicalRocksdbDumper{ cfg: cnf, } + } else if storageEngine == cst.StorageEngineTokudb { + dumper = &PhysicalTokudbDumper{ + cfg: cnf, + } } else { dumper = &PhysicalDumper{ cnf: cnf, diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_tokudb.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_tokudb.go new file mode 100644 index 0000000000..7be355622f --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_tokudb.go @@ -0,0 +1,255 @@ +package backupexe + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util" +) + +// PhysicalTokudbDumper physical tokudb dumper +type PhysicalTokudbDumper struct { + cfg *config.BackupConfig + backupLogfile string + dbbackupHome string + flushWaitTimeout int + mysqlVersion string + isOfficial bool + tokudbCmd string + storageEngine string + mysqlRole string + masterHost string + masterPort int + backupStartTime time.Time + backupEndTime time.Time + backupTargetPath string +} + +// buildArgs construct the instruction parameters for data recovery. +func (p *PhysicalTokudbDumper) buildArgs() []string { + // p.backupTargetPath is initialized in initConfig + args := []string{ + fmt.Sprintf("-u%s", p.cfg.Public.MysqlUser), + fmt.Sprintf("-p%s", p.cfg.Public.MysqlPasswd), + fmt.Sprintf("-h%s", p.cfg.Public.MysqlHost), + fmt.Sprintf("-P%d", p.cfg.Public.MysqlPort), + fmt.Sprintf("--flush-wait-timeout=%d", p.flushWaitTimeout), + } + if strings.ToLower(p.cfg.Public.MysqlRole) == cst.RoleSlave { + args = append(args, "--dump-slave") + } + args = append(args, fmt.Sprintf("%s", p.backupTargetPath)) + return args +} + +// initConfig init config +func (p *PhysicalTokudbDumper) initConfig(mysqlVersion string) error { + if p.cfg == nil { + return errors.New("tokudb physical dumper config missed") + } + if p.flushWaitTimeout == 0 { + p.flushWaitTimeout = 30 + } + cmdPath, err := os.Executable() + if err != nil { + return err + } + + p.dbbackupHome = filepath.Dir(cmdPath) + + // connect to the mysql and obtain the base information + db, err := mysqlconn.InitConn(&p.cfg.Public) + if err != nil { + logger.Log.Errorf("can not connect to the mysql, host:%s, port:%d, errmsg:%s", + p.cfg.Public.MysqlHost, p.cfg.Public.MysqlPort, err) + return err + } + defer func() { + _ = db.Close() + }() + + p.mysqlVersion, p.isOfficial = util.VersionParser(mysqlVersion) + p.storageEngine, err = mysqlconn.GetStorageEngine(db) + + if err != nil { + logger.Log.Errorf("can not get the storage engine from the mysql, host:%s, port:%d, errmsg:%s", + p.cfg.Public.MysqlHost, p.cfg.Public.MysqlPort, err) + return err + } + + // keep the storage engine name is lower + p.storageEngine = strings.ToLower(p.storageEngine) + p.mysqlRole = strings.ToLower(p.cfg.Public.MysqlRole) + + // if the current node is slave, obtain the master ip and port + if p.mysqlRole == cst.RoleSlave || p.mysqlRole == cst.RoleRepeater { + p.masterHost, p.masterPort, err = mysqlconn.ShowMysqlSlaveStatus(db) + if err != nil { + logger.Log.Errorf("can not get the master host and port from the mysql, host:%s, port:%d, errmsg:%s", + p.cfg.Public.MysqlHost, p.cfg.Public.MysqlPort, err) + return err + } + } + + p.backupTargetPath = filepath.Join(p.cfg.Public.BackupDir, p.cfg.Public.TargetName()) + p.tokudbCmd = filepath.Join("bin", cst.ToolTokudbBackup) + BackupTool = cst.ToolTokudbBackup + return nil +} + +// Execute Perform data recovery operations. +func (p *PhysicalTokudbDumper) Execute(enableTimeOut bool) error { + p.backupStartTime = time.Now() + defer func() { + p.backupEndTime = time.Now() + }() + + // the storage engine must be tokudb + if p.storageEngine != cst.StorageEngineTokudb { + err := fmt.Errorf("unsupported engine:%s, host:%s, port:%d", p.storageEngine, + p.cfg.Public.MysqlHost, p.cfg.Public.MysqlPort) + logger.Log.Error(err) + return err + } + + binPath := filepath.Join(p.dbbackupHome, p.tokudbCmd) + args := p.buildArgs() + + // perform the dump operation + var cmd *exec.Cmd + backupCmd := fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " ")) + + if enableTimeOut { + timeDiffUnix, err := GetMaxRunningTime(p.cfg.Public.BackupTimeOut) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUnix))*time.Second) + defer cancel() + + cmd = exec.CommandContext(ctx, "sh", "-c", backupCmd) + } else { + cmd = exec.Command("sh", "-c", backupCmd) + } + + // create a dumper log file to store the log of the dumper command + p.backupLogfile = fmt.Sprintf("dumper_%s_%d_%d.log", + p.storageEngine, p.cfg.Public.MysqlPort, int(time.Now().Weekday())) + p.backupLogfile = filepath.Join(p.dbbackupHome, "logs", p.backupLogfile) + + // pre-created dump log file + outFile, err := os.Create(p.backupLogfile) + + if err != nil { + logger.Log.Errorf("can not create the dumper log file, file name:%s, errmsg:%s", p.backupLogfile, err) + return err + } + + defer func() { + _ = outFile.Close() + }() + + // redirect standard output and error messages to a file + cmd.Stdout = outFile + cmd.Stderr = outFile + + // perform the dump command + p.backupStartTime = time.Now() + defer func() { + p.backupEndTime = time.Now() + }() + err = cmd.Run() + if err != nil { + logger.Log.Errorf("can not run the tokudb physical dumper command:%s, engine:%s, errmsg:%s", + backupCmd, p.storageEngine, err) + return err + } + + logger.Log.Infof("dump tokudb success, command:%s", cmd.String()) + return nil +} + +// PrepareBackupMetaInfo generate the metadata of database backup +func (p *PhysicalTokudbDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbareport.IndexContent, error) { + + // parse the binlog position + binlogInfoFileName := filepath.Join(p.backupTargetPath, "xtrabackup_binlog_info") + slaveInfoFileName := filepath.Join(p.backupTargetPath, "xtrabackup_slave_info") + tmpFileName := filepath.Join(p.backupTargetPath, "tmp_dbbackup_go.txt") + + // obtain the qpress command path + exepath, err := os.Executable() + if err != nil { + return nil, err + } + exepath = filepath.Dir(exepath) + + var metaInfo = dbareport.IndexContent{ + BinlogInfo: dbareport.BinlogStatusInfo{}, + } + + // parse the binlog + masterStatus, err := parseXtraBinlogInfo("", binlogInfoFileName, tmpFileName) + if err != nil { + logger.Log.Errorf("do not parse xtrabackup binlog file, file name:%s, errmsg:%s", + slaveInfoFileName, err) + return nil, err + } + + // save the master node status + metaInfo.BinlogInfo.ShowMasterStatus = masterStatus + metaInfo.BinlogInfo.ShowMasterStatus.MasterHost = cnf.Public.MysqlHost + metaInfo.BinlogInfo.ShowMasterStatus.MasterPort = cnf.Public.MysqlPort + + // parse the information of the master node + if p.mysqlRole == cst.RoleSlave || p.mysqlRole == cst.RoleRepeater { + slaveStatus, err := parseXtraSlaveInfo("", slaveInfoFileName, tmpFileName) + + if err != nil { + logger.Log.Errorf("do not parse xtrabackup slave information, xtrabackup file:%s, errmsg:%s", + slaveInfoFileName, err) + return nil, err + } + + metaInfo.BinlogInfo.ShowSlaveStatus = slaveStatus + metaInfo.BinlogInfo.ShowSlaveStatus.MasterHost = p.masterHost + metaInfo.BinlogInfo.ShowSlaveStatus.MasterPort = p.masterPort + } + + // parse xtrabackup_info + if fileTokudbBegin, err := os.ReadFile(filepath.Join(p.backupTargetPath, "TOKUDB.BEGIN")); err == nil { + metaInfo.BackupBeginTime, _ = time.ParseInLocation("20060102_150405", + strings.TrimSpace(string(fileTokudbBegin)), time.Local) + } else { + metaInfo.BackupBeginTime, _ = time.Parse(time.DateTime, p.backupStartTime.Format(time.DateTime)) + } + if fileTokudbEnd, err := os.ReadFile(filepath.Join(p.backupTargetPath, "TOKUDB.END")); err == nil { + metaInfo.BackupEndTime, _ = time.ParseInLocation("20060102_150405", + strings.TrimSpace(string(fileTokudbEnd)), time.Local) + } else { + metaInfo.BackupEndTime, _ = time.Parse(time.DateTime, p.backupEndTime.Format(time.DateTime)) + } + metaInfo.BackupConsistentTime = metaInfo.BackupEndTime + + // teh mark indicating whether the update is a full backup or not + metaInfo.JudgeIsFullBackup(&cnf.Public) + if err = os.Remove(tmpFileName); err != nil { + logger.Log.Errorf("do not delete the tmp file, file name:%s, errmsg:%s", tmpFileName, err) + return &metaInfo, err + } + + return &metaInfo, nil +} diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go index 688f146b1a..cdeb222dda 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go @@ -47,11 +47,14 @@ func BuildLoader(cnf *config.BackupConfig, backupType string, backupTool string, if err := validate.GoValidateStruct(cnf.PhysicalLoad, false, false); err != nil { return nil, err } - if cst.StorageEngineRocksdb == storageEngine { loader = &PhysicalRocksdbLoader{ cfg: cnf, } + } else if storageEngine == cst.StorageEngineTokudb || backupTool == "tokudb_back.pl" { + loader = &PhysicalTokudbLoader{ + cnf: cnf, + } } else { loader = &PhysicalLoader{ cnf: cnf, diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical_tokudb.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical_tokudb.go new file mode 100644 index 0000000000..e11f7e45c7 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical_tokudb.go @@ -0,0 +1,91 @@ +package backupexe + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util" + + "github.com/pkg/errors" +) + +// PhysicalTokudbLoader this is used to load physical backup +// decompress, apply, recover +type PhysicalTokudbLoader struct { + cnf *config.BackupConfig + dbbackupHome string + mysqlVersion string + storageEngine string +} + +func (p *PhysicalTokudbLoader) initConfig(indexContent *dbareport.IndexContent) error { + if p.cnf == nil { + return errors.New("tokudb loader params is nil") + } + if cmdPath, err := os.Executable(); err != nil { + return err + } else { + p.dbbackupHome = filepath.Dir(cmdPath) + } + + p.mysqlVersion, _ = util.VersionParser(indexContent.MysqlVersion) + p.storageEngine = strings.ToLower(indexContent.StorageEngine) + return nil +} + +// Execute excute loading backup with physical backup tool +func (p *PhysicalTokudbLoader) Execute() error { + if p.storageEngine != cst.StorageEngineTokudb { + err := fmt.Errorf("%s engine not supported", p.storageEngine) + logger.Log.Error(err) + return err + } + + err := p.load() + if err != nil { + return err + } + + return nil +} + +// load tokudb_recovery.pl +func (p *PhysicalTokudbLoader) load() error { + binPath := filepath.Join(p.dbbackupHome, "bin", "tokudb_recovery.pl") + + args := []string{fmt.Sprintf("--defaults-file=%s", p.cnf.PhysicalLoad.DefaultsFile)} + args = append(args, fmt.Sprintf("--backup-path=%s", p.cnf.PhysicalLoad.MysqlLoadDir)) + + // 日志输出到当前目录的 logs/loader_tokudb_xx.log + pwd, _ := os.Getwd() + logfile := filepath.Join(pwd, "logs", fmt.Sprintf("loader_%s_%d_%d.log", + p.storageEngine, p.cnf.Public.MysqlPort, int(time.Now().Weekday()))) + _ = os.MkdirAll(filepath.Dir(logfile), 0755) + + args = append(args, ">>", logfile, "2>&1") + logger.Log.Info("tokudb recover command:", binPath, strings.Join(args, " ")) + outStr, errStr, err := cmutil.ExecCommand(true, "", binPath, args...) + if err != nil { + logger.Log.Error("tokudb recover failed: ", err, errStr) + grepError := []string{"tail", "-5", logfile} + errStrPrefix := fmt.Sprintf("tail 5 error from %s", logfile) + errStrDetail, _, _ := cmutil.ExecCommand(true, "", grepError[0], grepError[1:]...) + if len(strings.TrimSpace(errStrDetail)) > 0 { + logger.Log.Info(errStrPrefix) + logger.Log.Error(errStrDetail) + } else { + logger.Log.Warn("can not find more detail error message from ", logfile) + } + return errors.WithMessagef(err, fmt.Sprintf("%s: %s\n%s", errStr, errStrPrefix, errStrDetail)) + } + logger.Log.Info("tokudb recover success: ", outStr) + return nil +}