[jobmanager] Recover from job panics

Signed-off-by: Dmitrii Okunev <xaionaro@meta.com>
linuxboot · Jul 4, 2023 · 73ffdb3 · 73ffdb3
1 parent fa98f00
commit 73ffdb3
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/pkg/jobmanager/jobmanager.go b/pkg/jobmanager/jobmanager.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/facebookincubator/go-belt/beltctx"
+	"github.com/facebookincubator/go-belt/tool/experimental/errmon"
 	"github.com/insomniacslk/xjson"
 
 	"github.com/linuxboot/contest/pkg/api"
@@ -183,6 +184,9 @@ loop:
 			logging.Debugf(ev.Context, "Handling event %+v", ev)
 			handlerWg.Add(1)
 			go func() {
+				defer func() {
+					errmon.ObserveRecoverCtx(ev.Context, recover())
+				}()
 				defer handlerWg.Done()
 				jm.handleEvent(ev)
 			}()

diff --git a/pkg/jobmanager/start.go b/pkg/jobmanager/start.go
@@ -102,6 +102,9 @@ func (jm *JobManager) startJob(ctx context.Context, j *job.Job, resumeState *job
 }
 
 func (jm *JobManager) runJob(ctx context.Context, j *job.Job, resumeState *job.PauseEventPayload) {
+	defer func() {
+		errmon.ObserveRecoverCtx(ctx, recover())
+	}()
 	defer func() {
 		jm.jobsMu.Lock()
 		delete(jm.jobs, j.ID)