From 625a5d202d3eb4d8b9987c89dd3d61800825be39 Mon Sep 17 00:00:00 2001 From: Aaron Hetherington Date: Tue, 15 Aug 2023 16:50:58 +0100 Subject: [PATCH] Expand retry conditions for K8 logs --- pkg/workceptor/kubernetes.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index 7c5340232..80e695c94 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -239,6 +239,7 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout streamReader := bufio.NewReader(logStream) for *stdinErr == nil { // check between every line read to see if we need to stop reading line, err := streamReader.ReadString('\n') + _, goaway := err.(http2.GoAwayError) if err == io.EOF { kw.Debug( "Detected EOF for pod %s/%s. Will retry %d more times. Error: %s", @@ -256,17 +257,18 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout } return - } else if _, ok := err.(http2.GoAwayError); ok { + } else if err != nil || goaway { // GOAWAY is sent by the server to indicate that the server is gracefully shutting down // this happens if the kube API server we are connected to is being restarted or is shutting down // for example during a cluster upgrade and rolling restart of the master node kw.Info( - "Detected http2.GoAwayError for pod %s/%s. Will retry %d more times. Error: %s", + "Detected Error: %s for pod %s/%s. Will retry %d more times.", + err, podNamespace, podName, remainingRetries, - err, ) + successfulWrite = false remainingRetries-- if remainingRetries > 0 { @@ -275,7 +277,7 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout break } } - if err != nil { + if err != nil && remainingRetries == 0 { *stdoutErr = err kw.Error("Error reading from pod %s/%s: %s", podNamespace, podName, err)