Added logics for quorum loss scenario.

config/rbac/role.yaml

@@ -7,151 +7,69 @@ metadata:
   name: manager-role
 rules:
 - apiGroups:
-  - ""
+  - batch
   resources:
-  - pods
+  - jobs
   verbs:
-  - list
-  - watch
+  - create
   - delete
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  - endpoints
-  verbs:
   - get
   - list
   - patch
   - update
   - watch
 - apiGroups:
-  - ""
+  - coordination.k8s.io
   resources:
-  - events
+  - leases
   verbs:
   - create
+  - delete
+  - deletecollection
   - get
   - list
-  - watch
   - patch
   - update
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - list
   - watch
-  - create
-  - update
-  - patch
-  - delete
 - apiGroups:
-  - rbac.authorization.k8s.io
+  - druid.gardener.cloud
   resources:
-  - roles
-  - rolebindings
+  - etcds
   verbs:
-  - get
-  - list
-  - watch
   - create
-  - update
-  - patch
   - delete
-- apiGroups:
-  - ""
-  - apps
-  resources:
-  - services
-  - configmaps
-  - statefulsets
-  verbs:
   - get
   - list
   - patch
   - update
   - watch
-  - create
-  - delete
-- apiGroups:
-  - batch
-  resources:
-  - jobs
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
-- apiGroups:
-  - batch
-  resources:
-  - cronjobs
-  verbs:
-  - get
-  - list
-  - watch
-  - delete
-- apiGroups:
-  - druid.gardener.cloud
-  resources:
-  - etcds
-  - etcdcopybackupstasks
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
 - apiGroups:
   - druid.gardener.cloud
   resources:
   - etcds/status
-  - etcds/finalizers
-  - etcdcopybackupstasks/status
-  - etcdcopybackupstasks/finalizers
   verbs:
   - get
-  - update
   - patch
-  - create
-- apiGroups:
-  - coordination.k8s.io
-  resources:
-  - leases
-  verbs:
-  - get
-  - list
-  - watch
-  - create
   - update
-  - patch
-  - delete
-  - deletecollection
 - apiGroups:
-  - ""
+  - druid.gardener.cloud
   resources:
-  - persistentvolumeclaims
+  - secrets
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - policy
   resources:
   - poddisruptionbudgets
   verbs:
+  - create
+  - delete
   - get
   - list
-  - watch
-  - create
-  - update
   - patch
-  - delete
+  - update
+  - watch

controllers/config/custodian.go

@@ -22,6 +22,8 @@ type EtcdCustodianController struct {
 	EtcdMember EtcdMemberConfig
 	// SyncPeriod is the duration after which re-enqueuing happens.
 	SyncPeriod time.Duration
+	// EnableAutomaticQuorumLossHandling is the flag to enable automatic handling of quorum loss. Set it false by default.
+	EnableAutomaticQuorumLossHandling bool
 }
 
 type EtcdMemberConfig struct {

controllers/controllers_suite_test.go

@@ -51,6 +51,7 @@ var (
 	mgrStopped *sync.WaitGroup
 
 	activeDeadlineDuration   time.Duration
+	waitDuration             time.Duration
 	backupCompactionSchedule = "15 */24 * * *"
 
 	revertFns []func()
@@ -102,7 +103,9 @@ var _ = BeforeSuite(func(done Done) {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
-	er, err := NewEtcdReconcilerWithImageVector(mgr, false)
+	waitDuration, err = time.ParseDuration("10s")
+	Expect(err).NotTo(HaveOccurred())
+	er, err := NewEtcdReconcilerWithImageVector(mgr, false, waitDuration)
 	Expect(err).NotTo(HaveOccurred())
 
 	err = er.SetupWithManager(mgr, 5, true)
@@ -114,8 +117,10 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).NotTo(HaveOccurred())
 
 	custodian := NewEtcdCustodian(mgr, controllersconfig.EtcdCustodianController{
+		SyncPeriod:                        10 * time.Second,
+		EnableAutomaticQuorumLossHandling: true,
 		EtcdMember: controllersconfig.EtcdMemberConfig{
-			EtcdMemberNotReadyThreshold: 1 * time.Minute,
+			EtcdMemberNotReadyThreshold: 20 * time.Second,
 		},
 	})
 
@@ -138,7 +143,7 @@ var _ = BeforeSuite(func(done Done) {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
-	err = lc.SetupWithManager(mgr, 1)
+	err = lc.SetupWithManager(mgr, 5)
 	Expect(err).NotTo(HaveOccurred())
 
 	mgrStopped = startTestManager(mgrCtx, mgr)