Reload config on update - Updated

cmds/houndd/main.go

@@ -1,60 +1,56 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/blang/semver/v4"
+	"github.com/fsnotify/fsnotify"
 	"log"
-	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 
-	"github.com/blang/semver/v4"
-	"github.com/hound-search/hound/api"
 	"github.com/hound-search/hound/config"
 	"github.com/hound-search/hound/searcher"
-	"github.com/hound-search/hound/ui"
 	"github.com/hound-search/hound/web"
 )
 
-const gracefulShutdownSignal = syscall.SIGTERM
-
 var (
 	info_log   *log.Logger
 	error_log  *log.Logger
 	_, b, _, _ = runtime.Caller(0)
 	basepath   = filepath.Dir(b)
 )
 
-func makeSearchers(cfg *config.Config) (map[string]*searcher.Searcher, bool, error) {
+func makeSearchers(cfg *config.Config, searchers map[string]*searcher.Searcher) (bool, error) {
 	// Ensure we have a dbpath
 	if _, err := os.Stat(cfg.DbPath); err != nil {
 		if err := os.MkdirAll(cfg.DbPath, os.ModePerm); err != nil {
-			return nil, false, err
+			return false, err
 		}
 	}
 
-	searchers, errs, err := searcher.MakeAll(cfg)
+	errs, err := searcher.MakeAll(cfg, searchers)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
 	if len(errs) > 0 {
 		// NOTE: This mutates the original config so the repos
 		// are not even seen by other code paths.
-		for name, _ := range errs { //nolint
+		for name := range errs {
 			delete(cfg.Repos, name)
 		}
 
-		return searchers, false, nil
+		return false, nil
 	}
 
-	return searchers, true, nil
+	return true, nil
 }
 
 func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.Searcher) {
@@ -73,46 +69,18 @@ func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.
 	}()
 }
 
-func registerShutdownSignal() <-chan os.Signal {
+func registerShutdownSignal() chan os.Signal {
 	shutdownCh := make(chan os.Signal, 1)
-	signal.Notify(shutdownCh, gracefulShutdownSignal)
+	signal.Notify(shutdownCh, syscall.SIGTERM)
+	signal.Notify(shutdownCh, syscall.SIGINT)
 	return shutdownCh
 }
 
-func makeTemplateData(cfg *config.Config) (interface{}, error) { //nolint
-	var data struct {
-		ReposAsJson string
-	}
-
-	res := map[string]*config.Repo{}
-	for name, repo := range cfg.Repos {
-		res[name] = repo
-	}
-
-	b, err := json.Marshal(res)
-	if err != nil {
-		return nil, err
-	}
-
-	data.ReposAsJson = string(b)
-	return &data, nil
-}
-
-func runHttp( //nolint
-	addr string,
-	dev bool,
-	cfg *config.Config,
-	idx map[string]*searcher.Searcher) error {
-	m := http.DefaultServeMux
-
-	h, err := ui.Content(dev, cfg)
-	if err != nil {
-		return err
+func unregisterShutdownSignal(shutdownCh chan os.Signal) {
+	if shutdownCh == nil {
+		return
 	}
-
-	m.Handle("/", h)
-	api.Setup(m, idx, cfg.ResultLimit)
-	return http.ListenAndServe(addr, m)
+	signal.Stop(shutdownCh)
 }
 
 // TODO: Automatically increment this when building a release
@@ -141,46 +109,88 @@ func main() {
 		os.Exit(0)
 	}
 
+	idx := make(map[string]*searcher.Searcher)
+
 	var cfg config.Config
-	if err := cfg.LoadFromFile(*flagConf); err != nil {
-		panic(err)
-	}
 
-	// Start the web server on a background routine.
-	ws := web.Start(&cfg, *flagAddr, *flagDev)
+	var shutdownCh chan os.Signal
+	shutdownCh = nil
+	var configUpdateLock sync.Mutex
 
-	// It's not safe to be killed during makeSearchers, so register the
-	// shutdown signal here and defer processing it until we are ready.
-	shutdownCh := registerShutdownSignal()
-	idx, ok, err := makeSearchers(&cfg)
-	if err != nil {
-		log.Panic(err)
-	}
-	if !ok {
-		info_log.Println("Some repos failed to index, see output above")
-	} else {
-		info_log.Println("All indexes built!")
+	loadConfig := func(server *web.Server) {
+		configUpdateLock.Lock()
+		defer configUpdateLock.Unlock()
+		// store existing cfg to check if it's changed
+		cfgJson, _ := cfg.ToJsonString()
+
+		if err := cfg.LoadFromFile(*flagConf); err != nil {
+			panic(err)
+		}
+		// unregister shutdown signal to create a new one
+		unregisterShutdownSignal(shutdownCh)
+		shutdownCh = nil
+
+		// It's not safe to be killed during makeSearchers, so register the
+		// shutdown signal here and defer processing it until we are ready.
+		shutdownCh = registerShutdownSignal()
+
+		ok, err := makeSearchers(&cfg, idx)
+		if err != nil {
+			log.Panic(err)
+		}
+		if !ok {
+			info_log.Println("Some repos failed to index, see output above")
+		} else {
+			info_log.Println("All indexes built!")
+		}
+		// handle shutdown signal now
+		handleShutdown(shutdownCh, idx)
+		if server != nil {
+			// if server has been passed, now check if cfg json has been changed
+			newCfgJson, err := cfg.ToJsonString()
+			if (err == nil) && (cfgJson != newCfgJson) {
+				// cfg json changed, update the server
+				info_log.Println("configJson updated, reloading server")
+				if err := server.UpdateServeWithIndex(idx); err != nil {
+					error_log.Printf("updating server failed for some reason: %s", err)
+				}
+			}
+		}
 	}
 
-	handleShutdown(shutdownCh, idx)
+	// Start the web server on a background routine.
+	ws := web.Start(&cfg, *flagAddr, *flagDev)
 
 	host := *flagAddr
 	if strings.HasPrefix(host, ":") { //nolint
 		host = "localhost" + host
 	}
+	info_log.Printf("started server without indexes at http://%s\n", host)
+
+	// Initial config load
+	loadConfig(nil)
+	info_log.Printf("loaded config")
 
 	if *flagDev {
 		info_log.Printf("[DEV] starting webpack-dev-server at localhost:8080...")
 		webpack := exec.Command("./node_modules/.bin/webpack-dev-server", "--mode", "development")
 		webpack.Dir = basepath + "/../../"
 		webpack.Stdout = os.Stdout
 		webpack.Stderr = os.Stderr
-		err = webpack.Start()
-		if err != nil {
+
+		if err := webpack.Start(); err != nil {
 			error_log.Println(err)
 		}
 	}
 
+	// watch for config file changes
+	configWatcher := config.NewWatcher(*flagConf)
+	configWatcher.OnChange(
+		func(fsnotify.Event) {
+			loadConfig(ws)
+		},
+	)
+
 	info_log.Printf("running server at http://%s\n", host)
 
 	// Fully enable the web server now that we have indexes

config/config.go

@@ -194,7 +194,9 @@ func (c *Config) LoadFromFile(filename string) error {
 		return err
 	}
 	defer r.Close()
-
+	// reset Repos and VCSConfigMessages so that upon reload we clear out deleted things
+	c.Repos = make(map[string]*Repo)
+	c.VCSConfigMessages = make(map[string]*SecretMessage)
 	if err := json.NewDecoder(r).Decode(c); err != nil {
 		return err
 	}

config/watcher.go

@@ -0,0 +1,79 @@
+package config
+
+import (
+	"log"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+// WatcherListenerFunc defines the signature for listner functions
+type WatcherListenerFunc func(fsnotify.Event)
+
+// Watcher watches for configuration updates and provides hooks for
+// triggering post events
+type Watcher struct {
+	listeners []WatcherListenerFunc
+}
+
+// NewWatcher returns a new file watcher
+func NewWatcher(cfgPath string) *Watcher {
+	log.Printf("setting up watcher for %s", cfgPath)
+	w := Watcher{}
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		watcher, err := fsnotify.NewWatcher()
+		if err != nil {
+			log.Panic(err)
+		}
+		defer watcher.Close()
+		// Event listener setup
+		eventWG := sync.WaitGroup{}
+		eventWG.Add(1)
+		go func() {
+			defer eventWG.Done()
+			for {
+				select {
+				case event, ok := <-watcher.Events:
+					if !ok {
+						// events channel is closed
+						log.Printf("error: events channel is closed\n")
+						return
+					}
+					// only trigger on creates and writes of the watched config file
+					if event.Name == cfgPath && event.Op&fsnotify.Write == fsnotify.Write {
+						log.Printf("change in config file (%s) detected\n", cfgPath)
+						for _, listener := range w.listeners {
+							listener(event)
+						}
+					}
+				case err, ok := <-watcher.Errors:
+					if !ok {
+						// errors channel is closed
+						log.Printf("error: errors channel is closed\n")
+						return
+					}
+					log.Println("error:", err)
+					return
+				}
+			}
+		}()
+		// add config file
+		if err := watcher.Add(cfgPath); err != nil {
+			log.Fatalf("failed to watch %s", cfgPath)
+		}
+		// setup is complete
+		wg.Done()
+		// wait for the event listener to complete before exiting
+		eventWG.Wait()
+	}()
+	// wait for watcher setup to complete
+	wg.Wait()
+	return &w
+}
+
+// OnChange registers a listener function to be called if a file changes
+func (w *Watcher) OnChange(listener WatcherListenerFunc) {
+	w.listeners = append(w.listeners, listener)
+}

go.mod

@@ -4,5 +4,6 @@ go 1.16
 
 require (
 	github.com/blang/semver/v4 v4.0.0
-	golang.org/x/mod v0.10.0
+	github.com/fsnotify/fsnotify v1.6.0
+	golang.org/x/mod v0.12.0
 )

go.sum

@@ -1,11 +1,15 @@
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -16,6 +20,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=