diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index eb8038a2..d9e9a733 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -1,29 +1,25 @@ package main import ( - "encoding/json" "flag" "fmt" + "github.com/blang/semver/v4" + "github.com/fsnotify/fsnotify" "log" - "net/http" "os" "os/exec" "os/signal" "path/filepath" "runtime" "strings" + "sync" "syscall" - "github.com/blang/semver/v4" - "github.com/hound-search/hound/api" "github.com/hound-search/hound/config" "github.com/hound-search/hound/searcher" - "github.com/hound-search/hound/ui" "github.com/hound-search/hound/web" ) -const gracefulShutdownSignal = syscall.SIGTERM - var ( info_log *log.Logger error_log *log.Logger @@ -31,30 +27,30 @@ var ( basepath = filepath.Dir(b) ) -func makeSearchers(cfg *config.Config) (map[string]*searcher.Searcher, bool, error) { +func makeSearchers(cfg *config.Config, searchers map[string]*searcher.Searcher) (bool, error) { // Ensure we have a dbpath if _, err := os.Stat(cfg.DbPath); err != nil { if err := os.MkdirAll(cfg.DbPath, os.ModePerm); err != nil { - return nil, false, err + return false, err } } - searchers, errs, err := searcher.MakeAll(cfg) + errs, err := searcher.MakeAll(cfg, searchers) if err != nil { - return nil, false, err + return false, err } if len(errs) > 0 { // NOTE: This mutates the original config so the repos // are not even seen by other code paths. - for name, _ := range errs { //nolint + for name := range errs { delete(cfg.Repos, name) } - return searchers, false, nil + return false, nil } - return searchers, true, nil + return true, nil } func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.Searcher) { @@ -73,46 +69,18 @@ func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher. }() } -func registerShutdownSignal() <-chan os.Signal { +func registerShutdownSignal() chan os.Signal { shutdownCh := make(chan os.Signal, 1) - signal.Notify(shutdownCh, gracefulShutdownSignal) + signal.Notify(shutdownCh, syscall.SIGTERM) + signal.Notify(shutdownCh, syscall.SIGINT) return shutdownCh } -func makeTemplateData(cfg *config.Config) (interface{}, error) { //nolint - var data struct { - ReposAsJson string - } - - res := map[string]*config.Repo{} - for name, repo := range cfg.Repos { - res[name] = repo - } - - b, err := json.Marshal(res) - if err != nil { - return nil, err - } - - data.ReposAsJson = string(b) - return &data, nil -} - -func runHttp( //nolint - addr string, - dev bool, - cfg *config.Config, - idx map[string]*searcher.Searcher) error { - m := http.DefaultServeMux - - h, err := ui.Content(dev, cfg) - if err != nil { - return err +func unregisterShutdownSignal(shutdownCh chan os.Signal) { + if shutdownCh == nil { + return } - - m.Handle("/", h) - api.Setup(m, idx, cfg.ResultLimit) - return http.ListenAndServe(addr, m) + signal.Stop(shutdownCh) } // TODO: Automatically increment this when building a release @@ -141,33 +109,67 @@ func main() { os.Exit(0) } + idx := make(map[string]*searcher.Searcher) + var cfg config.Config - if err := cfg.LoadFromFile(*flagConf); err != nil { - panic(err) - } - // Start the web server on a background routine. - ws := web.Start(&cfg, *flagAddr, *flagDev) + var shutdownCh chan os.Signal + shutdownCh = nil + var configUpdateLock sync.Mutex - // It's not safe to be killed during makeSearchers, so register the - // shutdown signal here and defer processing it until we are ready. - shutdownCh := registerShutdownSignal() - idx, ok, err := makeSearchers(&cfg) - if err != nil { - log.Panic(err) - } - if !ok { - info_log.Println("Some repos failed to index, see output above") - } else { - info_log.Println("All indexes built!") + loadConfig := func(server *web.Server) { + configUpdateLock.Lock() + defer configUpdateLock.Unlock() + // store existing cfg to check if it's changed + cfgJson, _ := cfg.ToJsonString() + + if err := cfg.LoadFromFile(*flagConf); err != nil { + panic(err) + } + // unregister shutdown signal to create a new one + unregisterShutdownSignal(shutdownCh) + shutdownCh = nil + + // It's not safe to be killed during makeSearchers, so register the + // shutdown signal here and defer processing it until we are ready. + shutdownCh = registerShutdownSignal() + + ok, err := makeSearchers(&cfg, idx) + if err != nil { + log.Panic(err) + } + if !ok { + info_log.Println("Some repos failed to index, see output above") + } else { + info_log.Println("All indexes built!") + } + // handle shutdown signal now + handleShutdown(shutdownCh, idx) + if server != nil { + // if server has been passed, now check if cfg json has been changed + newCfgJson, err := cfg.ToJsonString() + if (err == nil) && (cfgJson != newCfgJson) { + // cfg json changed, update the server + info_log.Println("configJson updated, reloading server") + if err := server.UpdateServeWithIndex(idx); err != nil { + error_log.Printf("updating server failed for some reason: %s", err) + } + } + } } - handleShutdown(shutdownCh, idx) + // Start the web server on a background routine. + ws := web.Start(&cfg, *flagAddr, *flagDev) host := *flagAddr if strings.HasPrefix(host, ":") { //nolint host = "localhost" + host } + info_log.Printf("started server without indexes at http://%s\n", host) + + // Initial config load + loadConfig(nil) + info_log.Printf("loaded config") if *flagDev { info_log.Printf("[DEV] starting webpack-dev-server at localhost:8080...") @@ -175,12 +177,20 @@ func main() { webpack.Dir = basepath + "/../../" webpack.Stdout = os.Stdout webpack.Stderr = os.Stderr - err = webpack.Start() - if err != nil { + + if err := webpack.Start(); err != nil { error_log.Println(err) } } + // watch for config file changes + configWatcher := config.NewWatcher(*flagConf) + configWatcher.OnChange( + func(fsnotify.Event) { + loadConfig(ws) + }, + ) + info_log.Printf("running server at http://%s\n", host) // Fully enable the web server now that we have indexes diff --git a/config/config.go b/config/config.go index c978c009..bbd53d49 100644 --- a/config/config.go +++ b/config/config.go @@ -194,7 +194,9 @@ func (c *Config) LoadFromFile(filename string) error { return err } defer r.Close() - + // reset Repos and VCSConfigMessages so that upon reload we clear out deleted things + c.Repos = make(map[string]*Repo) + c.VCSConfigMessages = make(map[string]*SecretMessage) if err := json.NewDecoder(r).Decode(c); err != nil { return err } diff --git a/config/watcher.go b/config/watcher.go new file mode 100644 index 00000000..bd3a0486 --- /dev/null +++ b/config/watcher.go @@ -0,0 +1,79 @@ +package config + +import ( + "log" + "sync" + + "github.com/fsnotify/fsnotify" +) + +// WatcherListenerFunc defines the signature for listner functions +type WatcherListenerFunc func(fsnotify.Event) + +// Watcher watches for configuration updates and provides hooks for +// triggering post events +type Watcher struct { + listeners []WatcherListenerFunc +} + +// NewWatcher returns a new file watcher +func NewWatcher(cfgPath string) *Watcher { + log.Printf("setting up watcher for %s", cfgPath) + w := Watcher{} + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Panic(err) + } + defer watcher.Close() + // Event listener setup + eventWG := sync.WaitGroup{} + eventWG.Add(1) + go func() { + defer eventWG.Done() + for { + select { + case event, ok := <-watcher.Events: + if !ok { + // events channel is closed + log.Printf("error: events channel is closed\n") + return + } + // only trigger on creates and writes of the watched config file + if event.Name == cfgPath && event.Op&fsnotify.Write == fsnotify.Write { + log.Printf("change in config file (%s) detected\n", cfgPath) + for _, listener := range w.listeners { + listener(event) + } + } + case err, ok := <-watcher.Errors: + if !ok { + // errors channel is closed + log.Printf("error: errors channel is closed\n") + return + } + log.Println("error:", err) + return + } + } + }() + // add config file + if err := watcher.Add(cfgPath); err != nil { + log.Fatalf("failed to watch %s", cfgPath) + } + // setup is complete + wg.Done() + // wait for the event listener to complete before exiting + eventWG.Wait() + }() + // wait for watcher setup to complete + wg.Wait() + return &w +} + +// OnChange registers a listener function to be called if a file changes +func (w *Watcher) OnChange(listener WatcherListenerFunc) { + w.listeners = append(w.listeners, listener) +} diff --git a/go.mod b/go.mod index 3d5151e6..1cf2d011 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,6 @@ go 1.16 require ( github.com/blang/semver/v4 v4.0.0 - golang.org/x/mod v0.10.0 + github.com/fsnotify/fsnotify v1.6.0 + golang.org/x/mod v0.12.0 ) diff --git a/go.sum b/go.sum index ab72a812..1ece7cc7 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,15 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -16,6 +20,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/searcher/searcher.go b/searcher/searcher.go index 791ce810..93c36b5d 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -52,7 +52,7 @@ type limiter chan bool */ type foundRefs struct { refs []*index.IndexRef - claimed map[*index.IndexRef]bool + claimed map[string]bool lock sync.Mutex } @@ -89,7 +89,7 @@ func (r *foundRefs) claim(ref *index.IndexRef) { r.lock.Lock() defer r.lock.Unlock() - r.claimed[ref] = true + r.claimed[ref.Dir()] = true } /** @@ -101,7 +101,7 @@ func (r *foundRefs) removeUnclaimed() error { defer r.lock.Unlock() for _, ref := range r.refs { - if r.claimed[ref] { + if r.claimed[ref.Dir()] { continue } @@ -223,7 +223,7 @@ func findExistingRefs(dbpath string) (*foundRefs, error) { return &foundRefs{ refs: refs, - claimed: map[*index.IndexRef]bool{}, + claimed: map[string]bool{}, }, nil } @@ -264,7 +264,7 @@ func reportOnMemory() { // Utility function for producing a hex encoded sha1 hash for a string. func hashFor(name string) string { h := sha1.New() - h.Write([]byte(name)) //nolint + h.Write([]byte(name)) //nolint return hex.EncodeToString(h.Sum(nil)) } @@ -282,24 +282,34 @@ func init() { // occurred and no other return values are valid. If an error occurs that is specific // to a particular searcher, that searcher will not be present in the searcher map and // will have an error entry in the error map. -func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) { +func MakeAll(cfg *config.Config, searchers map[string]*Searcher) (map[string]error, error) { errs := map[string]error{} - searchers := map[string]*Searcher{} refs, err := findExistingRefs(cfg.DbPath) if err != nil { - return nil, nil, err + return nil, err } lim := makeLimiter(cfg.MaxConcurrentIndexers) - n := len(cfg.Repos) + n := 0 + for name := range cfg.Repos { + if s, ok := searchers[name]; ok { + // claim any already running searcher refs so that they don't get removed + refs.claim(s.idx.Ref) + continue + } + n++ + } // Channel to receive the results from newSearcherConcurrent function. resultCh := make(chan searcherResult, n) // Start new searchers for all repos in different go routines while // respecting cfg.MaxConcurrentIndexers. for name, repo := range cfg.Repos { + if _, ok := searchers[name]; ok { + continue + } go newSearcherConcurrent(cfg.DbPath, name, repo, refs, lim, resultCh) } @@ -315,7 +325,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) } if err := refs.removeUnclaimed(); err != nil { - return nil, nil, err + return nil, err } // after all the repos are in good shape, we start their polling @@ -323,7 +333,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) s.begin() } - return searchers, errs, nil + return errs, nil } // Creates a new Searcher that is available for searches as soon as this returns. @@ -407,7 +417,6 @@ func newSearcher( return nil, err } - rev, err := wd.PullOrClone(vcsDir, repo.Url) if err != nil { return nil, err diff --git a/web/web.go b/web/web.go index d32dd4e8..bd7647e9 100644 --- a/web/web.go +++ b/web/web.go @@ -70,6 +70,16 @@ func Start(cfg *config.Config, addr string, dev bool) *Server { // ServeWithIndex allow the server to start offering the search UI and the // search APIs operating on the given indexes. func (s *Server) ServeWithIndex(idx map[string]*searcher.Searcher) error { + err := s.UpdateServeWithIndex(idx) + if err != nil { + return err + } + + return <-s.ch +} + +// UpdateServeWithIndex updates the server handler without returning the error channel +func (s *Server) UpdateServeWithIndex(idx map[string]*searcher.Searcher) error { h, err := ui.Content(s.dev, s.cfg) if err != nil { return err @@ -81,5 +91,5 @@ func (s *Server) ServeWithIndex(idx map[string]*searcher.Searcher) error { s.serveWith(m) - return <-s.ch + return nil }