Merge pull request #462 from vshn/improve-monitoring

Clean histories for successful and failed jobs separately

.codeclimate.yml

@@ -17,6 +17,7 @@ exclude_patterns:
 - 'features/'
 - '**/node_modules/'
 - 'script/'
+- 'envtest/'
 - '**/spec/'
 - '**/test/'
 - '**/tests/'

Makefile

@@ -13,7 +13,7 @@ include Makefile.vars.mk
 e2e_make := $(MAKE) -C e2e
 go_build ?= go build -o $(BIN_FILENAME) main.go
 
-setup-envtest ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest
+setup-envtest ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest $(ENVTEST_ADDITIONAL_FLAGS)
 
 # Run tests (see https://sdk.operatorframework.io/docs/building-operators/golang/references/envtest-setup)
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin

Makefile.vars.mk

@@ -27,6 +27,8 @@ E2E_REPO ?= local.dev/k8up/e2e
 E2E_IMG = $(E2E_REPO):$(E2E_TAG)
 BATS_FILES ?= .
 
+ENVTEST_ADDITIONAL_FLAGS ?=
+
 INTEGRATION_TEST_DEBUG_OUTPUT ?= false
 
 KUSTOMIZE ?= go run sigs.k8s.io/kustomize/kustomize/v3

api/v1alpha1/archive_types.go

@@ -76,6 +76,33 @@ func (a *Archive) GetResources() corev1.ResourceRequirements {
 	return a.Spec.Resources
 }
 
+// GetFailedJobsHistoryLimit returns failed jobs history limit.
+// Returns KeepJobs if unspecified.
+func (a *Archive) GetFailedJobsHistoryLimit() *int {
+	if a.Spec.FailedJobsHistoryLimit != nil {
+		return a.Spec.FailedJobsHistoryLimit
+	}
+	return a.Spec.KeepJobs
+}
+
+// GetSuccessfulJobsHistoryLimit returns successful jobs history limit.
+// Returns KeepJobs if unspecified.
+func (a *Archive) GetSuccessfulJobsHistoryLimit() *int {
+	if a.Spec.SuccessfulJobsHistoryLimit != nil {
+		return a.Spec.SuccessfulJobsHistoryLimit
+	}
+	return a.Spec.KeepJobs
+}
+
+// GetJobObjects returns a sortable list of jobs
+func (a *ArchiveList) GetJobObjects() JobObjectList {
+	items := make(JobObjectList, len(a.Items))
+	for i := range a.Items {
+		items[i] = &a.Items[i]
+	}
+	return items
+}
+
 // GetDeepCopy returns a deep copy
 func (in *ArchiveSchedule) GetDeepCopy() ScheduleSpecInterface {
 	return in.DeepCopy()

api/v1alpha1/backup_types.go

@@ -12,8 +12,19 @@ import (
 type BackupSpec struct {
 	RunnableSpec `json:",inline"`
 
-	// KeepJobs amount of jobs to keep for later analysis
+	// KeepJobs amount of jobs to keep for later analysis.
+	//
+	// Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
+	// +optional
 	KeepJobs *int `json:"keepJobs,omitempty"`
+	// FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
+	// KeepJobs is used property is not specified.
+	// +optional
+	FailedJobsHistoryLimit *int `json:"failedJobsHistoryLimit,omitempty"`
+	// SuccessfulJobsHistoryLimit amount of successful jobs to keep for later analysis.
+	// KeepJobs is used property is not specified.
+	// +optional
+	SuccessfulJobsHistoryLimit *int `json:"successfulJobsHistoryLimit,omitempty"`
 
 	// PromURL sets a prometheus push URL where the backup container send metrics to
 	// +optional
@@ -95,6 +106,33 @@ func (b *Backup) GetResources() corev1.ResourceRequirements {
 	return b.Spec.Resources
 }
 
+// GetFailedJobsHistoryLimit returns failed jobs history limit.
+// Returns KeepJobs if unspecified.
+func (b *Backup) GetFailedJobsHistoryLimit() *int {
+	if b.Spec.FailedJobsHistoryLimit != nil {
+		return b.Spec.FailedJobsHistoryLimit
+	}
+	return b.Spec.KeepJobs
+}
+
+// GetSuccessfulJobsHistoryLimit returns successful jobs history limit.
+// Returns KeepJobs if unspecified.
+func (b *Backup) GetSuccessfulJobsHistoryLimit() *int {
+	if b.Spec.SuccessfulJobsHistoryLimit != nil {
+		return b.Spec.SuccessfulJobsHistoryLimit
+	}
+	return b.Spec.KeepJobs
+}
+
+// GetJobObjects returns a sortable list of jobs
+func (b *BackupList) GetJobObjects() JobObjectList {
+	items := make(JobObjectList, len(b.Items))
+	for i := range b.Items {
+		items[i] = &b.Items[i]
+	}
+	return items
+}
+
 // GetDeepCopy returns a deep copy
 func (in *BackupSchedule) GetDeepCopy() ScheduleSpecInterface {
 	return in.DeepCopy()

api/v1alpha1/check_types.go

@@ -15,8 +15,19 @@ type CheckSpec struct {
 	// +optional
 	PromURL string `json:"promURL,omitempty"`
 
-	// KeepJobs amount of jobs to keep for later analysis
+	// KeepJobs amount of jobs to keep for later analysis.
+	//
+	// Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
+	// +optional
 	KeepJobs *int `json:"keepJobs,omitempty"`
+	// FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
+	// KeepJobs is used property is not specified.
+	// +optional
+	FailedJobsHistoryLimit *int `json:"failedJobsHistoryLimit,omitempty"`
+	// SuccessfulJobsHistoryLimit amount of successful jobs to keep for later analysis.
+	// KeepJobs is used property is not specified.
+	// +optional
+	SuccessfulJobsHistoryLimit *int `json:"successfulJobsHistoryLimit,omitempty"`
 }
 
 // +kubebuilder:object:root=true
@@ -74,6 +85,33 @@ func (c *Check) GetResources() corev1.ResourceRequirements {
 	return c.Spec.Resources
 }
 
+// GetFailedJobsHistoryLimit returns failed jobs history limit.
+// Returns KeepJobs if unspecified.
+func (c *Check) GetFailedJobsHistoryLimit() *int {
+	if c.Spec.FailedJobsHistoryLimit != nil {
+		return c.Spec.FailedJobsHistoryLimit
+	}
+	return c.Spec.KeepJobs
+}
+
+// GetSuccessfulJobsHistoryLimit returns successful jobs history limit.
+// Returns KeepJobs if unspecified.
+func (c *Check) GetSuccessfulJobsHistoryLimit() *int {
+	if c.Spec.SuccessfulJobsHistoryLimit != nil {
+		return c.Spec.SuccessfulJobsHistoryLimit
+	}
+	return c.Spec.KeepJobs
+}
+
+// GetJobObjects returns a sortable list of jobs
+func (c *CheckList) GetJobObjects() JobObjectList {
+	items := make(JobObjectList, len(c.Items))
+	for i := range c.Items {
+		items[i] = &c.Items[i]
+	}
+	return items
+}
+
 // GetDeepCopy returns a deep copy
 func (in *CheckSchedule) GetDeepCopy() ScheduleSpecInterface {
 	return in.DeepCopy()

api/v1alpha1/history_limits_test.go

@@ -0,0 +1,59 @@
+package v1alpha1_test
+
+import (
+	"testing"
+
+	"github.com/vshn/k8up/api/v1alpha1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type limiter interface {
+	GetSuccessfulJobsHistoryLimit() *int
+	GetFailedJobsHistoryLimit() *int
+}
+
+var historyLimitTestCases = map[string]func(successful, failed, deprecatedKeep *int) limiter{
+	"Archive": func(successful, failed, deprecatedKeep *int) limiter {
+		return &v1alpha1.Archive{Spec: v1alpha1.ArchiveSpec{RestoreSpec: &v1alpha1.RestoreSpec{SuccessfulJobsHistoryLimit: successful, FailedJobsHistoryLimit: failed, KeepJobs: deprecatedKeep}}}
+	},
+	"Backup": func(successful, failed, deprecatedKeep *int) limiter {
+		return &v1alpha1.Backup{Spec: v1alpha1.BackupSpec{SuccessfulJobsHistoryLimit: successful, FailedJobsHistoryLimit: failed, KeepJobs: deprecatedKeep}}
+	},
+	"Check": func(successful, failed, deprecatedKeep *int) limiter {
+		return &v1alpha1.Check{Spec: v1alpha1.CheckSpec{SuccessfulJobsHistoryLimit: successful, FailedJobsHistoryLimit: failed, KeepJobs: deprecatedKeep}}
+	},
+	"Prune": func(successful, failed, deprecatedKeep *int) limiter {
+		return &v1alpha1.Prune{Spec: v1alpha1.PruneSpec{SuccessfulJobsHistoryLimit: successful, FailedJobsHistoryLimit: failed, KeepJobs: deprecatedKeep}}
+	},
+	"Restore": func(successful, failed, deprecatedKeep *int) limiter {
+		return &v1alpha1.Restore{Spec: v1alpha1.RestoreSpec{SuccessfulJobsHistoryLimit: successful, FailedJobsHistoryLimit: failed, KeepJobs: deprecatedKeep}}
+	},
+}
+
+func TestHistoryLimits(t *testing.T) {
+	for name, createSpec := range historyLimitTestCases {
+		t.Run(name, func(t *testing.T) {
+			failedLimit := 1
+			successLimit := 2
+			keepJobs := 3
+
+			t.Run("JobsHistoryLimit", func(t *testing.T) {
+				limits := createSpec(&successLimit, &failedLimit, &keepJobs)
+				assert.Equal(t, *limits.GetFailedJobsHistoryLimit(), failedLimit)
+				assert.Equal(t, *limits.GetSuccessfulJobsHistoryLimit(), successLimit)
+			})
+			t.Run("fallback to deprecated KeepJobs", func(t *testing.T) {
+				limits := createSpec(nil, nil, &keepJobs)
+				assert.Equal(t, *limits.GetFailedJobsHistoryLimit(), keepJobs)
+				assert.Equal(t, *limits.GetSuccessfulJobsHistoryLimit(), keepJobs)
+			})
+			t.Run("no fallback value", func(t *testing.T) {
+				limits := createSpec(nil, nil, nil)
+				var nilInt *int
+				assert.Equal(t, limits.GetFailedJobsHistoryLimit(), nilInt)
+				assert.Equal(t, limits.GetSuccessfulJobsHistoryLimit(), nilInt)
+			})
+		})
+	}
+}

api/v1alpha1/job_object.go

@@ -0,0 +1,34 @@
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// +k8s:deepcopy-gen=false
+
+// JobObject is an interface that must be implemented by all CRDs that implement a job.
+type JobObject interface {
+	GetMetaObject() metav1.Object
+	GetRuntimeObject() runtime.Object
+	GetStatus() Status
+	SetStatus(s Status)
+	GetType() JobType
+	GetResources() corev1.ResourceRequirements
+}
+
+// +k8s:deepcopy-gen=false
+
+// JobObjectList is a sortable list of job objects
+type JobObjectList []JobObject
+
+func (jo JobObjectList) Len() int      { return len(jo) }
+func (jo JobObjectList) Swap(i, j int) { jo[i], jo[j] = jo[j], jo[i] }
+
+func (jo JobObjectList) Less(i, j int) bool {
+	if jo[i].GetMetaObject().GetCreationTimestamp().Time.Equal(jo[j].GetMetaObject().GetCreationTimestamp().Time) {
+		return jo[i].GetMetaObject().GetName() < jo[j].GetMetaObject().GetName()
+	}
+	return jo[i].GetMetaObject().GetCreationTimestamp().Time.Before(jo[j].GetMetaObject().GetCreationTimestamp().Time)
+}

api/v1alpha1/object_list_test.go

@@ -0,0 +1,82 @@
+package v1alpha1_test
+
+import (
+	"testing"
+
+	"github.com/vshn/k8up/api/v1alpha1"
+
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestGetObjectList(t *testing.T) {
+
+	type jobObjectList interface {
+		GetJobObjects() v1alpha1.JobObjectList
+	}
+
+	testCases := map[string]struct {
+		desc       string
+		createList func(itemName1, itemName2 string) jobObjectList
+	}{
+		"Archive": {
+			createList: func(itemName1, itemName2 string) jobObjectList {
+				return &v1alpha1.ArchiveList{
+					Items: []v1alpha1.Archive{
+						{ObjectMeta: v1.ObjectMeta{Name: itemName1}},
+						{ObjectMeta: v1.ObjectMeta{Name: itemName2}},
+					},
+				}
+			},
+		},
+		"Backup": {
+			createList: func(itemName1, itemName2 string) jobObjectList {
+				return &v1alpha1.BackupList{
+					Items: []v1alpha1.Backup{
+						{ObjectMeta: v1.ObjectMeta{Name: itemName1}},
+						{ObjectMeta: v1.ObjectMeta{Name: itemName2}},
+					},
+				}
+			},
+		},
+		"Check": {
+			createList: func(itemName1, itemName2 string) jobObjectList {
+				return &v1alpha1.CheckList{
+					Items: []v1alpha1.Check{
+						{ObjectMeta: v1.ObjectMeta{Name: itemName1}},
+						{ObjectMeta: v1.ObjectMeta{Name: itemName2}},
+					},
+				}
+			},
+		},
+		"Prune": {
+			createList: func(itemName1, itemName2 string) jobObjectList {
+				return &v1alpha1.PruneList{
+					Items: []v1alpha1.Prune{
+						{ObjectMeta: v1.ObjectMeta{Name: itemName1}},
+						{ObjectMeta: v1.ObjectMeta{Name: itemName2}},
+					},
+				}
+			},
+		},
+		"Restore": {
+			createList: func(itemName1, itemName2 string) jobObjectList {
+				return &v1alpha1.RestoreList{
+					Items: []v1alpha1.Restore{
+						{ObjectMeta: v1.ObjectMeta{Name: itemName1}},
+						{ObjectMeta: v1.ObjectMeta{Name: itemName2}},
+					},
+				}
+			},
+		},
+	}
+	for name, tC := range testCases {
+		t.Run(name, func(t *testing.T) {
+			name1 := "obj1"
+			name2 := "obj2"
+			list := tC.createList(name1, name2).GetJobObjects()
+			assert.Equal(t, name1, list[0].GetMetaObject().GetName())
+			assert.Equal(t, name2, list[1].GetMetaObject().GetName())
+		})
+	}
+}