From eadb12811ac699ca4a2a249c403e211ba8af22e0 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 20 Nov 2024 12:41:25 +0530 Subject: [PATCH] feat: fix potential deadlock in make primary Signed-off-by: Manan Gupta --- go/vt/vttablet/tabletserver/health_streamer.go | 4 +++- go/vt/vttablet/tabletserver/health_streamer_test.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index f9f65d197b2..eaeba6315e3 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -293,8 +293,10 @@ func (hs *healthStreamer) SetUnhealthyThreshold(v time.Duration) { // so it can read and write to the MySQL instance for schema-tracking. func (hs *healthStreamer) MakePrimary(serving bool) { hs.fieldsMu.Lock() - defer hs.fieldsMu.Unlock() hs.isServingPrimary = serving + // We let go of the lock here because we don't want to hold the lock when calling RegisterNotifier. + // If we keep holding the lock, there is a potential deadlock that can happen. + hs.fieldsMu.Unlock() // We register for notifications from the schema Engine only when schema tracking is enabled, // and we are going to a serving primary state. if serving && hs.signalWhenSchemaChange { diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index 3421141ff80..9561518eed6 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -592,13 +592,14 @@ func TestDeadlockBwCloseAndReload(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) - // Try running Close and reload in parallel multiple times. + // Try running Close & MakePrimary and reload in parallel multiple times. // This reproduces the deadlock quite readily. go func() { defer wg.Done() for i := 0; i < 100; i++ { hs.Close() hs.Open() + hs.MakePrimary(true) } }()