fix: not a real fix, simply extend the timeout to 24 hours

massix · Jun 29, 2024 · ab03319 · ab03319
1 parent 4d84775
commit ab03319
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -3,7 +3,7 @@ TERRAFORM := $(shell which terraform)
 DOCKER := $(shell which docker)
 APPNAME ?= chaos-monkey
 IMAGE ?= chaos-monkey
-TAG ?= 2.0.1
+TAG ?= 2.0.3
 
 all: bin/$(APPNAME)
 .PHONY: clean generate bin/$(APPNAME) image-version cluster-test

diff --git a/internal/watcher/crd.go b/internal/watcher/crd.go
@@ -75,13 +75,17 @@ func (c *CrdWatcher) Start(ctx context.Context) error {
 	var err error
 	var wg sync.WaitGroup
 
+	watchTimeout := int64((24 * time.Hour).Seconds())
 	w, err := c.ChaosMonkeyConfigurationInterface.Watch(ctx, metav1.ListOptions{
-		Watch: true,
+		Watch:          true,
+		TimeoutSeconds: &watchTimeout,
 	})
 	if err != nil {
 		return err
 	}
 
+	defer w.Stop()
+
 	c.setRunning(true)
 
 	for c.IsRunning() {

diff --git a/internal/watcher/namespace.go b/internal/watcher/namespace.go
@@ -74,11 +74,17 @@ func (n *NamespaceWatcher) Start(ctx context.Context) error {
 
 	logrus.Infof("Starting namespace watcher for %s", n.RootNamespace)
 
-	w, err := n.Watch(ctx, v1.ListOptions{})
+	timeoutSeconds := int64((24 * time.Hour).Seconds())
+	w, err := n.Watch(ctx, v1.ListOptions{
+		Watch:          true,
+		TimeoutSeconds: &timeoutSeconds,
+	})
 	if err != nil {
 		return err
 	}
 
+	defer w.Stop()
+
 	n.setRunning(true)
 
 	for n.IsRunning() {

diff --git a/internal/watcher/pod.go b/internal/watcher/pod.go
@@ -99,9 +99,12 @@ func (p *PodWatcher) IsRunning() bool {
 func (p *PodWatcher) Start(ctx context.Context) error {
 	var err error
 	logrus.Infof("Starting pod watcher in namespace %s", p.Namespace)
+
+	watchTimeout := int64((24 * time.Hour).Seconds())
 	w, err := p.Watch(ctx, metav1.ListOptions{
-		Watch:         true,
-		LabelSelector: p.LabelSelector,
+		Watch:          true,
+		LabelSelector:  p.LabelSelector,
+		TimeoutSeconds: &watchTimeout,
 	})
 	if err != nil {
 		return err
@@ -116,14 +119,10 @@ func (p *PodWatcher) Start(ctx context.Context) error {
 
 	for p.IsRunning() {
 		select {
-		case evt, ok := <-w.ResultChan():
+		case evt := <-w.ResultChan():
 			logrus.Debugf("Pod Watcher received event: %s", evt.Type)
 			pod := evt.Object.(*apicorev1.Pod)
 
-			if !ok {
-				return errors.New("Pod Watcher channel closed")
-			}
-
 			switch evt.Type {
 			case "", watch.Error:
 				logrus.Errorf("Received empty event or error from pod watcher: %+v", evt)