Skip to content

Commit

Permalink
feat: implement configurable behavior for the nswatcher
Browse files Browse the repository at this point in the history
  • Loading branch information
massix committed Jul 5, 2024
1 parent b792fa7 commit d3d4c96
Show file tree
Hide file tree
Showing 12 changed files with 511 additions and 140 deletions.
42 changes: 33 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ Basically, it spawns a new [goroutine](https://go.dev/tour/concurrency/1) with a
[CRD Watcher](#crd-watcher) every time a new namespace is detected and it stops the
corresponding goroutine when a namespace is deleted.

In the future, there will be the possibility to blacklist (or whitelist) some namespaces
depending on an annotation.
The Namespace can be [configured](#configuration) to either monitor all namespaces by
default (with an opt-out strategy) or to monitor only the namespaces which contain the
label `cm.massix.github.io/namespace="true"`. Check the [Configuration](#configuration)
paragraph for more details.

### CRD Watcher
We make use of a
Expand Down Expand Up @@ -178,13 +180,35 @@ spec:
```

## Configuration
The only configuration possible for the ChaosMonkey is setting the minimum log level,
this is done by setting the environment variable `CHAOSMONKEY_LOGLEVEL` to one of the
following values: `trace`, `debug`, `info`, `warn`, `error`, `critical` or `panic`.

The value is not case-sensitive.

Invalid or empty values will make ChaosMonkey default to the `info` level.
There are two configurable parts of the ChaosMonkey (on top of what the [CRD](./crds/chaosmonkey-configuration.yaml)
already permits of course).

**Minimum Log Level**: this is configurable using the environment variable `CHAOSMONKEY_LOGLEVEL`,
it accepts the following self explaining values: `trace`, `debug`, `info`, `warn`, `error`,
`critical` or `panic` and it sets the minimum log level for all the logging of the ChaosMonkey.

The value is not case-sensitive, invalid or empty values will make ChaosMonkey default to
the `info` level.

**Default Behavior**: this is used to configure the way the [Namespace Watcher](#namespace-watcher) should
behave in regards of additions and modifications of namespaces and it uses the environment
variable `CHAOSMONKEY_BEHAVIOR`. It currently accepts two values: `AllowAll` or `DenyAll`
(not case sensitive).

Setting it to `AllowAll` means that by default all namespaces are monitored, if
you want to opt-out a namespace you **must** create a new label in the
metadata of the namespace: `cm.massix.github.io/namespace="false"`, this will
make the Watcher ignore that namespace. All values which are not the string
`false` will cause the Watcher to take that namespace into account.

Setting it to `DenyAll` means that by default all namespaces are ignored, if
you want to opt-in a namespace you **must** create a new label in
the metadata of the namespace: `cm.massix.github.io/namespace="true"`, this will
make the Watcher take that namespace into account. All values which are not
the string `true` will cause the Watcher to ignore that namespace.

Injecting an incorrect value or no value at all will have ChaosMonkey use its
default behavior: `AllowAll`.

## Observability
The Chaos Monkey exposes some metrics using the [Prometheus](https://prometheus.io/) library and format.
Expand Down
13 changes: 11 additions & 2 deletions cmd/chaosmonkey/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func main() {
log := logrus.WithFields(logrus.Fields{"component": "main"})

// Get the LogLevel from the environment variable
ll, err := configuration.FromEnvironment()
ll, err := configuration.LogrusLevelFromEnvironment()
if err != nil {
log.Warnf("No loglevel provided, using default: %s", logrus.GetLevel())
} else {
Expand All @@ -48,10 +48,19 @@ func main() {
panic(err)
}

log.Info("Configuring default behavior via environment variable")
behavior, err := configuration.BehaviorFromEnvironment()
if err != nil {
log.Warnf("Error while configuring default behavior: %s", err)

behavior = configuration.AllowAll
log.Warnf("Using default behavior: %s", behavior)
}

clientset := kubernetes.NewForConfigOrDie(cfg)
cmcClientset := versioned.NewForConfigOrDie(cfg)

nsWatcher := watcher.DefaultNamespaceFactory(clientset, cmcClientset, nil, namespace)
nsWatcher := watcher.DefaultNamespaceFactory(clientset, cmcClientset, nil, namespace, behavior)

// Hook signals
s := make(chan os.Signal, 1)
Expand Down
21 changes: 21 additions & 0 deletions internal/configuration/behavior.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package configuration

import (
"errors"
"os"
"strings"
)

func BehaviorFromEnvironment() (Behavior, error) {
if val, ok := os.LookupEnv("CHAOSMONKEY_BEHAVIOR"); ok {
val = strings.ToUpper(val)

if val == string(AllowAll) || val == string(DenyAll) {
return Behavior(val), nil
} else {
return "", &InvalidBehavior{providedBehaviour: val}
}
}

return "", errors.New("No environment variable provided")
}
58 changes: 58 additions & 0 deletions internal/configuration/behavior_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package configuration_test

import (
"fmt"
"strings"
"testing"

"github.com/massix/chaos-monkey/internal/configuration"
)

func TestBehavior_ParseFromEnvironment(t *testing.T) {
goodEnvValues := map[string]configuration.Behavior{
"allowall": configuration.AllowAll,
"ALLOWALL": configuration.AllowAll,
"AllOwAll": configuration.AllowAll,
"denyall": configuration.DenyAll,
"DenyAll": configuration.DenyAll,
"DENYALL": configuration.DenyAll,
}

for key, val := range goodEnvValues {
t.Run(fmt.Sprintf("Can parse from environment (%s)", key), func(t *testing.T) {
t.Setenv("CHAOSMONKEY_BEHAVIOR", key)

if b, err := configuration.BehaviorFromEnvironment(); err != nil {
t.Error(err)
} else if b != val {
t.Errorf("Was expecting %s, got %s instead", val, b)
}
})
}

badEnvValues := []string{
"",
"invalid",
"geckos",
}

for _, val := range badEnvValues {
t.Run(fmt.Sprintf("It fails for invalid strings (%s)", val), func(t *testing.T) {
t.Setenv("CHAOSMONKEY_BEHAVIOR", val)

if b, err := configuration.BehaviorFromEnvironment(); err == nil {
t.Errorf("Was expecting error, received %s instead", b)
} else if err.Error() != fmt.Sprintf("Invalid behaviour: %s", strings.ToUpper(val)) {
t.Error(err)
}
})
}

t.Run("It fails if there is no environment variable", func(t *testing.T) {
if b, err := configuration.BehaviorFromEnvironment(); err == nil {
t.Errorf("Was expecting error, received %s instead", b)
} else if err.Error() != "No environment variable provided" {
t.Error(err)
}
})
}
61 changes: 61 additions & 0 deletions internal/configuration/logruslevel.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package configuration

import (
"errors"
"os"
"slices"
"strings"

"github.com/sirupsen/logrus"
)

func LogrusLevelFromEnvironment() (LogrusLevel, error) {
if val, ok := os.LookupEnv("CHAOSMONKEY_LOGLEVEL"); ok {
if newLevel, err := NewLogrusLevel(strings.ToLower(val)); err == nil {
return newLevel, nil
} else {
return "", err
}
}

return "", errors.New("No environment variable for configuring the log level found.")
}

func NewLogrusLevel(level string) (LogrusLevel, error) {
validLevels := []string{
"panic",
"fatal",
"error",
"warn",
"info",
"debug",
"trace",
}

if slices.Contains(validLevels, level) {
return LogrusLevel(level), nil
}

return "", &InvalidLogrusLevel{level}
}

func (l LogrusLevel) LogrusLevel() logrus.Level {
switch l {
case "panic":
return logrus.PanicLevel
case "fatal":
return logrus.FatalLevel
case "error":
return logrus.ErrorLevel
case "warn":
return logrus.WarnLevel
case "info":
return logrus.InfoLevel
case "debug":
return logrus.DebugLevel
case "trace":
return logrus.TraceLevel
default:
return logrus.InfoLevel
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package configuration_test

import (
"encoding/json"
"fmt"
"strings"
"testing"
Expand All @@ -10,10 +9,6 @@ import (
"github.com/sirupsen/logrus"
)

type UnmarshalTest struct {
Level configuration.LogrusLevel `json:"level"`
}

func Test_LogrusLevel(t *testing.T) {
t.Run("Can create a logrus level", func(t *testing.T) {
t.Parallel()
Expand All @@ -40,43 +35,14 @@ func Test_LogrusLevel(t *testing.T) {
t.Fatal(err)
}
})

t.Run("Can unmarshal a logrus level", func(t *testing.T) {
t.Parallel()

var unmarshalTest UnmarshalTest
err := json.Unmarshal([]byte(`{ "level": "trace" }`), &unmarshalTest)
if err != nil {
t.Fatal(err)
}

if unmarshalTest.Level.LogrusLevel() != logrus.TraceLevel {
t.Fatal(unmarshalTest.Level.LogrusLevel())
}
})

t.Run("Will fail if level is not valid", func(t *testing.T) {
t.Parallel()

var unmarshalTest UnmarshalTest
err := json.Unmarshal([]byte(`{ "level": "invalid" }`), &unmarshalTest)
if err == nil {
t.Fatal("Was expecting error")
}

if err.Error() != "Invalid logrus level: invalid" {
t.Fatal(err)
}
})
}

func TestLogLevel_FromEnvironment(t *testing.T) {
for _, level := range []string{"PANIC", "FaTaL", "eRROR", "WARN", "info", "debug", "trace"} {
t.Logf("Testing with loglevel: %s", level)
t.Run(fmt.Sprintf("Can set loglevel from environment (%s)", level), func(t *testing.T) {
t.Setenv("CHAOSMONKEY_LOGLEVEL", level)

ll, err := configuration.FromEnvironment()
ll, err := configuration.LogrusLevelFromEnvironment()
if err != nil {
t.Fatal(err)
}
Expand All @@ -88,11 +54,10 @@ func TestLogLevel_FromEnvironment(t *testing.T) {
}

for _, level := range []string{"", "invalid", "geckos"} {
t.Logf("Testing with loglevel: %s", level)
t.Run(fmt.Sprintf("It fails for invalid strings (%s)", level), func(t *testing.T) {
t.Setenv("CHAOSMONKEY_LOGLEVEL", level)

ll, err := configuration.FromEnvironment()
ll, err := configuration.LogrusLevelFromEnvironment()
if err == nil || ll != "" {
t.Fatalf("Was not expecting to succeed: %s", ll)
}
Expand All @@ -107,7 +72,7 @@ func TestLogLevel_FromEnvironment(t *testing.T) {
}

t.Run("It fails if there is no environment variable", func(t *testing.T) {
ll, err := configuration.FromEnvironment()
ll, err := configuration.LogrusLevelFromEnvironment()
if err == nil || ll != "" {
t.Fatal("Was not expecting to succeed")
}
Expand Down
Loading

0 comments on commit d3d4c96

Please sign in to comment.