From 005b58d7056f103588e718260dc647ca84316483 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Tue, 30 Jan 2024 14:41:24 +0200 Subject: [PATCH 1/9] K8SPXC-1152: restore stucks on operator restart https://perconadev.atlassian.net/browse/K8SPXC-1152 --- ...cona.com_perconaxtradbclusterrestores.yaml | 5 + ...pxc.percona.com_perconaxtradbclusters.yaml | 537 +++++++++++++---- deploy/bundle.yaml | 542 ++++++++++++++---- deploy/crd.yaml | 542 ++++++++++++++---- deploy/cw-bundle.yaml | 542 ++++++++++++++---- pkg/apis/pxc/v1/pxc_prestore_types.go | 10 +- pkg/controller/pxcrestore/controller.go | 498 ++++++---------- pkg/controller/pxcrestore/restore.go | 71 +-- pkg/controller/pxcrestore/restorer.go | 102 ++-- pkg/controller/pxcrestore/util.go | 210 +++++++ pkg/k8s/setowner.go | 10 - 11 files changed, 2161 insertions(+), 908 deletions(-) create mode 100644 pkg/controller/pxcrestore/util.go diff --git a/config/crd/bases/pxc.percona.com_perconaxtradbclusterrestores.yaml b/config/crd/bases/pxc.percona.com_perconaxtradbclusterrestores.yaml index df177af350..61c72a236d 100644 --- a/config/crd/bases/pxc.percona.com_perconaxtradbclusterrestores.yaml +++ b/config/crd/bases/pxc.percona.com_perconaxtradbclusterrestores.yaml @@ -347,6 +347,11 @@ spec: type: object status: properties: + allowUnsafeConfigurations: + type: boolean + clusterSize: + format: int32 + type: integer comments: type: string completed: diff --git a/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml b/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml index 3e57fa69b1..a0f5e1196a 100644 --- a/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml +++ b/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml @@ -263,6 +263,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -329,6 +339,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -393,6 +413,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -459,6 +489,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -895,18 +935,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -949,6 +977,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -1089,6 +1119,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1155,6 +1195,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1219,6 +1269,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1285,6 +1345,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1511,6 +1581,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -1561,6 +1639,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -1936,18 +2022,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -1990,6 +2064,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -2044,6 +2120,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -2303,18 +2390,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -2357,6 +2432,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -2543,6 +2620,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -2909,6 +3022,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2959,6 +3080,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -3520,18 +3649,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -3574,6 +3691,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -3978,6 +4097,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4044,6 +4173,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4108,6 +4247,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4174,6 +4323,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4373,6 +4532,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -4423,6 +4590,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -4790,18 +4965,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -4844,6 +5007,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -4898,6 +5063,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -5157,18 +5333,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -5211,6 +5375,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -5397,6 +5563,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -5763,6 +5965,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5813,6 +6023,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -6374,18 +6592,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -6428,6 +6634,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -6562,6 +6770,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -6628,6 +6846,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -6692,6 +6920,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -6758,6 +6996,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -6959,6 +7207,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7009,6 +7265,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7409,18 +7673,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -7463,6 +7715,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -7517,6 +7771,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -7776,18 +8041,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -7830,6 +8083,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8016,6 +8271,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -8382,6 +8673,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -8432,6 +8731,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -8993,18 +9300,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -9047,6 +9342,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 62eddbbb1a..9e1079dd0b 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -588,6 +588,11 @@ spec: type: object status: properties: + allowUnsafeConfigurations: + type: boolean + clusterSize: + format: int32 + type: integer comments: type: string completed: @@ -1165,6 +1170,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1231,6 +1246,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1295,6 +1320,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1361,6 +1396,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1797,18 +1842,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -1851,6 +1884,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -1991,6 +2026,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2057,6 +2102,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2121,6 +2176,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2187,6 +2252,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2413,6 +2488,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2463,6 +2546,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2838,18 +2929,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -2892,6 +2971,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -2946,6 +3027,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -3205,18 +3297,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -3259,6 +3339,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -3445,6 +3527,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -3811,6 +3929,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -3861,6 +3987,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -4422,18 +4556,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -4476,6 +4598,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -4880,6 +5004,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4946,6 +5080,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5010,6 +5154,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5076,6 +5230,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5275,6 +5439,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5325,6 +5497,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5692,18 +5872,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -5746,6 +5914,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -5800,6 +5970,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -6059,18 +6240,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -6113,6 +6282,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -6299,6 +6470,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -6665,6 +6872,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -6715,6 +6930,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7276,18 +7499,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -7330,6 +7541,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -7464,6 +7677,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7530,6 +7753,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7594,6 +7827,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7660,6 +7903,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7861,6 +8114,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7911,6 +8172,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -8311,18 +8580,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -8365,6 +8622,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8419,6 +8678,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -8678,18 +8948,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -8732,6 +8990,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8918,6 +9178,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -9284,6 +9580,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -9334,6 +9638,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -9895,18 +10207,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -9949,6 +10249,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 70a5dd0921..a4ce872a4b 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -588,6 +588,11 @@ spec: type: object status: properties: + allowUnsafeConfigurations: + type: boolean + clusterSize: + format: int32 + type: integer comments: type: string completed: @@ -1165,6 +1170,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1231,6 +1246,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1295,6 +1320,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1361,6 +1396,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1797,18 +1842,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -1851,6 +1884,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -1991,6 +2026,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2057,6 +2102,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2121,6 +2176,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2187,6 +2252,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2413,6 +2488,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2463,6 +2546,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2838,18 +2929,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -2892,6 +2971,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -2946,6 +3027,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -3205,18 +3297,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -3259,6 +3339,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -3445,6 +3527,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -3811,6 +3929,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -3861,6 +3987,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -4422,18 +4556,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -4476,6 +4598,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -4880,6 +5004,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4946,6 +5080,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5010,6 +5154,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5076,6 +5230,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5275,6 +5439,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5325,6 +5497,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5692,18 +5872,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -5746,6 +5914,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -5800,6 +5970,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -6059,18 +6240,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -6113,6 +6282,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -6299,6 +6470,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -6665,6 +6872,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -6715,6 +6930,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7276,18 +7499,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -7330,6 +7541,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -7464,6 +7677,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7530,6 +7753,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7594,6 +7827,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7660,6 +7903,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7861,6 +8114,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7911,6 +8172,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -8311,18 +8580,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -8365,6 +8622,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8419,6 +8678,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -8678,18 +8948,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -8732,6 +8990,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8918,6 +9178,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -9284,6 +9580,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -9334,6 +9638,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -9895,18 +10207,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -9949,6 +10249,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index fcb1efc52a..ae078b6c68 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -588,6 +588,11 @@ spec: type: object status: properties: + allowUnsafeConfigurations: + type: boolean + clusterSize: + format: int32 + type: integer comments: type: string completed: @@ -1165,6 +1170,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1231,6 +1246,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1295,6 +1320,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1361,6 +1396,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -1797,18 +1842,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -1851,6 +1884,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -1991,6 +2026,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2057,6 +2102,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2121,6 +2176,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2187,6 +2252,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -2413,6 +2488,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2463,6 +2546,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -2838,18 +2929,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -2892,6 +2971,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -2946,6 +3027,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -3205,18 +3297,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -3259,6 +3339,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -3445,6 +3527,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -3811,6 +3929,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -3861,6 +3987,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -4422,18 +4556,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -4476,6 +4598,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -4880,6 +5004,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -4946,6 +5080,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5010,6 +5154,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5076,6 +5230,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -5275,6 +5439,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5325,6 +5497,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -5692,18 +5872,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -5746,6 +5914,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -5800,6 +5970,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -6059,18 +6240,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -6113,6 +6282,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -6299,6 +6470,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -6665,6 +6872,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -6715,6 +6930,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7276,18 +7499,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -7330,6 +7541,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -7464,6 +7677,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7530,6 +7753,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7594,6 +7827,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7660,6 +7903,16 @@ spec: type: string type: object type: object + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic namespaceSelector: properties: matchExpressions: @@ -7861,6 +8114,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -7911,6 +8172,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -8311,18 +8580,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -8365,6 +8622,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8419,6 +8678,17 @@ spec: - type type: object type: array + currentVolumeAttributesClassName: + type: string + modifyVolumeStatus: + properties: + status: + type: string + targetVolumeAttributesClassName: + type: string + required: + - status + type: object phase: type: string type: object @@ -8678,18 +8948,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -8732,6 +8990,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: @@ -8918,6 +9178,42 @@ spec: sources: items: properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object configMap: properties: items: @@ -9284,6 +9580,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -9334,6 +9638,14 @@ spec: required: - port type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object tcpSocket: properties: host: @@ -9895,18 +10207,6 @@ spec: type: object resources: properties: - claims: - items: - properties: - name: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -9949,6 +10249,8 @@ spec: type: object storageClassName: type: string + volumeAttributesClassName: + type: string volumeMode: type: string volumeName: diff --git a/pkg/apis/pxc/v1/pxc_prestore_types.go b/pkg/apis/pxc/v1/pxc_prestore_types.go index accd547816..c82e482d31 100644 --- a/pkg/apis/pxc/v1/pxc_prestore_types.go +++ b/pkg/apis/pxc/v1/pxc_prestore_types.go @@ -19,10 +19,12 @@ type PerconaXtraDBClusterRestoreSpec struct { // PerconaXtraDBClusterRestoreStatus defines the observed state of PerconaXtraDBClusterRestore type PerconaXtraDBClusterRestoreStatus struct { - State BcpRestoreStates `json:"state,omitempty"` - Comments string `json:"comments,omitempty"` - CompletedAt *metav1.Time `json:"completed,omitempty"` - LastScheduled *metav1.Time `json:"lastscheduled,omitempty"` + State BcpRestoreStates `json:"state,omitempty"` + Comments string `json:"comments,omitempty"` + CompletedAt *metav1.Time `json:"completed,omitempty"` + LastScheduled *metav1.Time `json:"lastscheduled,omitempty"` + PXCSize int32 `json:"clusterSize,omitempty"` + AllowUnsafeConfig bool `json:"allowUnsafeConfigurations,omitempty"` } type PITR struct { diff --git a/pkg/controller/pxcrestore/controller.go b/pkg/controller/pxcrestore/controller.go index 87c8bf0a81..dbfe8ff262 100644 --- a/pkg/controller/pxcrestore/controller.go +++ b/pkg/controller/pxcrestore/controller.go @@ -3,15 +3,13 @@ package pxcrestore import ( "context" "fmt" - "strings" "time" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" + batchv1 "k8s.io/api/batch/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" k8sretry "k8s.io/client-go/util/retry" @@ -24,8 +22,6 @@ import ( "github.com/percona/percona-xtradb-cluster-operator/clientcmd" api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" - "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app" - "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset" "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup" "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup/storage" "github.com/percona/percona-xtradb-cluster-operator/version" @@ -93,10 +89,12 @@ type ReconcilePerconaXtraDBClusterRestore struct { func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logf.FromContext(ctx) - rr := reconcile.Result{} + rr := reconcile.Result{ + RequeueAfter: time.Second * 5, + } cr := &api.PerconaXtraDBClusterRestore{} - err := r.client.Get(context.TODO(), request.NamespacedName, cr) + err := r.client.Get(ctx, request.NamespacedName, cr) if err != nil { if k8serrors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. @@ -105,370 +103,240 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re // Error reading the object - requeue the request. return rr, err } - if cr.Status.State != api.RestoreNew { - return rr, nil - } - log.Info("backup restore request") - - err = r.setStatus(cr, api.RestoreStarting, "") - if err != nil { - return rr, errors.Wrap(err, "set status") - } - rJobsList := &api.PerconaXtraDBClusterRestoreList{} - err = r.client.List( - context.TODO(), - rJobsList, - &client.ListOptions{ - Namespace: cr.Namespace, - }, - ) - if err != nil { - return rr, errors.Wrap(err, "get restore jobs list") + switch cr.Status.State { + case api.RestoreSucceeded, api.RestoreFailed: + return reconcile.Result{}, nil } - returnMsg := fmt.Sprintf(backupRestoredMsg, cr.Name, cr.Spec.PXCCluster, cr.Name) + statusState := cr.Status.State + statusMsg := "" defer func() { - status := api.BcpRestoreStates(api.RestoreSucceeded) - if err != nil { - status = api.RestoreFailed - returnMsg = err.Error() - } - err := r.setStatus(cr, status, returnMsg) - if err != nil { - return + if err := setStatus(ctx, r.client, cr, statusState, statusMsg); err != nil { + log.Error(err, "failed to set status") } }() - for _, j := range rJobsList.Items { - if j.Spec.PXCCluster == cr.Spec.PXCCluster && - j.Name != cr.Name && j.Status.State != api.RestoreFailed && - j.Status.State != api.RestoreSucceeded { - err = errors.Errorf("unable to continue, concurent restore job %s running now.", j.Name) - return rr, err - } - } - - err = cr.CheckNsetDefaults() + otherRestore, err := isOtherRestoreInProgress(ctx, r.client, cr) if err != nil { - return rr, err + return rr, errors.Wrap(err, "failed to check if other restore is in progress") } - - cluster := new(api.PerconaXtraDBCluster) - err = r.client.Get(context.TODO(), types.NamespacedName{Name: cr.Spec.PXCCluster, Namespace: cr.Namespace}, cluster) - if err != nil { - err = errors.Wrapf(err, "get cluster %s", cr.Spec.PXCCluster) + if otherRestore != nil { + err = errors.Errorf("unable to continue, concurent restore job %s running now", otherRestore.Name) + statusState = api.RestoreFailed + statusMsg = err.Error() return rr, err } - clusterOrig := cluster.DeepCopy() - - err = cluster.CheckNSetDefaults(r.serverVersion, log) - if err != nil { - return reconcile.Result{}, fmt.Errorf("wrong PXC options: %v", err) - } - - err = backup.CheckPITRErrors(ctx, r.client, r.clientcmd, cluster) - if err != nil { - return reconcile.Result{}, err - } - - bcp, err := r.getBackup(ctx, cr) - if err != nil { - return rr, errors.Wrap(err, "get backup") - } - - annotations := cr.GetAnnotations() - _, unsafePITR := annotations[api.AnnotationUnsafePITR] - cond := meta.FindStatusCondition(bcp.Status.Conditions, api.BackupConditionPITRReady) - if cond != nil && cond.Status == metav1.ConditionFalse && !unsafePITR { - msg := fmt.Sprintf("Backup doesn't guarantee consistent recovery with PITR. Annotate PerconaXtraDBClusterRestore with %s to force it.", api.AnnotationUnsafePITR) - err = errors.New(msg) - return reconcile.Result{}, nil - } - err = r.validate(ctx, cr, bcp, cluster) - if err != nil { - err = errors.Wrap(err, "failed to validate restore job") + if err := cr.CheckNsetDefaults(); err != nil { + statusState = api.RestoreFailed + statusMsg = err.Error() return rr, err } - log.Info("stopping cluster", "cluster", cr.Spec.PXCCluster) - err = r.setStatus(cr, api.RestoreStopCluster, "") - if err != nil { - err = errors.Wrap(err, "set status") - return rr, err - } - err = r.stopCluster(cluster.DeepCopy()) - if err != nil { - err = errors.Wrapf(err, "stop cluster %s", cluster.Name) - return rr, err + cluster := new(api.PerconaXtraDBCluster) + if err := r.client.Get(ctx, types.NamespacedName{Name: cr.Spec.PXCCluster, Namespace: cr.Namespace}, cluster); err != nil { + if k8serrors.IsNotFound(err) { + statusState = api.RestoreFailed + statusMsg = err.Error() + } + return rr, errors.Wrapf(err, "get cluster %s", cr.Spec.PXCCluster) } - log.Info("starting restore", "cluster", cr.Spec.PXCCluster, "backup", cr.Spec.BackupName) - err = r.setStatus(cr, api.RestoreRestore, "") - if err != nil { - err = errors.Wrap(err, "set status") - return rr, err + if err := cluster.CheckNSetDefaults(r.serverVersion, log); err != nil { + statusState = api.RestoreFailed + statusMsg = err.Error() + return rr, errors.Wrap(err, "wrong PXC options") } - err = r.restore(ctx, cr, bcp, cluster) + err = backup.CheckPITRErrors(ctx, r.client, r.clientcmd, cluster) if err != nil { - err = errors.Wrap(err, "run restore") + statusState = api.RestoreFailed + statusMsg = err.Error() return rr, err } - log.Info("starting cluster", "cluster", cr.Spec.PXCCluster) - err = r.setStatus(cr, api.RestoreStartCluster, "") + bcp, err := getBackup(ctx, r.client, cr) if err != nil { - err = errors.Wrap(err, "set status") - return rr, err + statusState = api.RestoreFailed + statusMsg = err.Error() + return rr, errors.Wrap(err, "get backup") } - if cr.Spec.PITR != nil { - oldSize := cluster.Spec.PXC.Size - oldUnsafe := cluster.Spec.AllowUnsafeConfig - cluster.Spec.PXC.Size = 1 - cluster.Spec.AllowUnsafeConfig = true - - if err := r.startCluster(cluster); err != nil { - return rr, errors.Wrap(err, "restart cluster for pitr") + switch statusState { + case api.RestoreNew: + annotations := cr.GetAnnotations() + _, unsafePITR := annotations[api.AnnotationUnsafePITR] + cond := meta.FindStatusCondition(bcp.Status.Conditions, api.BackupConditionPITRReady) + if cond != nil && cond.Status == metav1.ConditionFalse && !unsafePITR { + statusState = api.RestoreFailed + statusMsg = fmt.Sprintf("Backup doesn't guarantee consistent recovery with PITR. Annotate PerconaXtraDBClusterRestore with %s to force it.", api.AnnotationUnsafePITR) + return rr, nil } - - log.Info("point-in-time recovering", "cluster", cr.Spec.PXCCluster) - err = r.setStatus(cr, api.RestorePITR, "") + err = r.validate(ctx, cr, bcp, cluster) if err != nil { - return rr, errors.Wrap(err, "set status") + if errors.Is(err, errWaitValidate) { + return rr, nil + } + err = errors.Wrap(err, "failed to validate restore job") + return rr, err } - - err = r.pitr(ctx, cr, bcp, cluster) + cr.Status.PXCSize = cluster.Spec.PXC.Size + cr.Status.AllowUnsafeConfig = cluster.Spec.AllowUnsafeConfig + log.Info("stopping cluster", "cluster", cr.Spec.PXCCluster) + statusState = api.RestoreStopCluster + case api.RestoreStopCluster: + err = stopCluster(ctx, r.client, cluster.DeepCopy()) if err != nil { - return rr, errors.Wrap(err, "run pitr") + switch err { + case errWaitingPods, errWaitingPVC: + log.Info("waiting for cluster to stop", "cluster", cr.Spec.PXCCluster, "msg", err.Error()) + return rr, nil + } + return rr, errors.Wrapf(err, "stop cluster %s", cluster.Name) } - cluster.Spec.PXC.Size = oldSize - cluster.Spec.AllowUnsafeConfig = oldUnsafe - - log.Info("starting cluster", "cluster", cr.Spec.PXCCluster) - err = r.setStatus(cr, api.RestoreStartCluster, "") + log.Info("starting restore", "cluster", cr.Spec.PXCCluster, "backup", cr.Spec.BackupName) + err = r.restore(ctx, cr, bcp, cluster) if err != nil { - err = errors.Wrap(err, "set status") + if errors.Is(err, errWaitInit) { + return rr, nil + } + err = errors.Wrap(err, "run restore") return rr, err } - } - - err = r.startCluster(clusterOrig) - if err != nil { - err = errors.Wrap(err, "restart cluster") - return rr, err - } - - log.Info(returnMsg) - - return rr, err -} - -func (r *ReconcilePerconaXtraDBClusterRestore) getBackup(ctx context.Context, cr *api.PerconaXtraDBClusterRestore) (*api.PerconaXtraDBClusterBackup, error) { - if cr.Spec.BackupSource != nil { - status := cr.Spec.BackupSource.DeepCopy() - status.State = api.BackupSucceeded - status.CompletedAt = nil - status.LastScheduled = nil - return &api.PerconaXtraDBClusterBackup{ - ObjectMeta: metav1.ObjectMeta{ - Name: cr.Name, - Namespace: cr.Namespace, - }, - Spec: api.PXCBackupSpec{ - PXCCluster: cr.Spec.PXCCluster, - StorageName: cr.Spec.BackupSource.StorageName, - }, - Status: *status, - }, nil - } - - bcp := &api.PerconaXtraDBClusterBackup{} - err := r.client.Get(ctx, types.NamespacedName{Name: cr.Spec.BackupName, Namespace: cr.Namespace}, bcp) - if err != nil { - err = errors.Wrapf(err, "get backup %s", cr.Spec.BackupName) - return bcp, err - } - if bcp.Status.State != api.BackupSucceeded { - err = errors.Errorf("backup %s didn't finished yet, current state: %s", bcp.Name, bcp.Status.State) - return bcp, err - } - - return bcp, nil -} - -const backupRestoredMsg = `You can view xtrabackup log: -$ kubectl logs job/restore-job-%s-%s -If everything is fine, you can cleanup the job: -$ kubectl delete pxc-restore/%s -` - -func (r *ReconcilePerconaXtraDBClusterRestore) stopCluster(c *api.PerconaXtraDBCluster) error { - var gracePeriodSec int64 - - if c.Spec.PXC != nil && c.Spec.PXC.TerminationGracePeriodSeconds != nil { - gracePeriodSec = int64(c.Spec.PXC.Size) * *c.Spec.PXC.TerminationGracePeriodSeconds - } - - patch := client.MergeFrom(c.DeepCopy()) - c.Spec.Pause = true - err := r.client.Patch(context.TODO(), c, patch) - if err != nil { - return errors.Wrap(err, "shutdown pods") - } - - ls := statefulset.NewNode(c).Labels() - err = r.waitForPodsShutdown(ls, c.Namespace, gracePeriodSec) - if err != nil { - return errors.Wrap(err, "shutdown pods") - } - - pvcs := corev1.PersistentVolumeClaimList{} - err = r.client.List( - context.TODO(), - &pvcs, - &client.ListOptions{ - Namespace: c.Namespace, - LabelSelector: labels.SelectorFromSet(ls), - }, - ) - if err != nil { - return errors.Wrap(err, "get pvc list") - } - - pxcNode := statefulset.NewNode(c) - pvcNameTemplate := app.DataVolumeName + "-" + pxcNode.StatefulSet().Name - for _, pvc := range pvcs.Items { - // check prefix just in case, to be sure we're not going to delete a wrong pvc - if pvc.Name == pvcNameTemplate+"-0" || !strings.HasPrefix(pvc.Name, pvcNameTemplate) { - continue + statusState = api.RestoreRestore + case api.RestoreRestore: + restorer, err := r.getRestorer(cr, bcp, cluster) + if err != nil { + return rr, errors.Wrap(err, "failed to get restorer") } - - err = r.client.Delete(context.TODO(), &pvc) + restorerJob, err := restorer.Job() if err != nil { - return errors.Wrap(err, "delete pvc") + return rr, errors.Wrap(err, "failed to create restore job") } - } - - err = r.waitForPVCShutdown(ls, c.Namespace) - if err != nil { - return errors.Wrap(err, "shutdown pvc") - } - - return nil -} - -func (r *ReconcilePerconaXtraDBClusterRestore) startCluster(cr *api.PerconaXtraDBCluster) (err error) { - // tryin several times just to avoid possible conflicts with the main controller - err = k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { - // need to get the object with latest version of meta-data for update - current := &api.PerconaXtraDBCluster{} - rerr := r.client.Get(context.TODO(), types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, current) - if rerr != nil { - return errors.Wrap(err, "get cluster") + job := new(batchv1.Job) + if err := r.client.Get(ctx, types.NamespacedName{ + Name: restorerJob.Name, + Namespace: restorerJob.Namespace, + }, job); err != nil { + return rr, errors.Wrap(err, "failed to get restore job") } - current.Spec = cr.Spec - return r.client.Update(context.TODO(), current) - }) - if err != nil { - return errors.Wrap(err, "update cluster") - } - - // give time for process new state - time.Sleep(10 * time.Second) - var waitLimit int32 = 2 * 60 * 60 // 2 hours - if cr.Spec.PXC.LivenessInitialDelaySeconds != nil { - waitLimit = *cr.Spec.PXC.LivenessInitialDelaySeconds * cr.Spec.PXC.Size - } - - for i := int32(0); i < waitLimit; i++ { - current := &api.PerconaXtraDBCluster{} - err = r.client.Get(context.TODO(), types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, current) + finished, err := isJobFinished(job) if err != nil { - return errors.Wrap(err, "get cluster") + statusState = api.RestoreFailed + statusMsg = err.Error() + return rr, err } - if current.Status.ObservedGeneration == current.Generation && current.Status.PXC.Status == api.AppStateReady { - return nil + if !finished { + log.Info("Waiting for restore job to finish", "job", job.Name) + return rr, nil } - time.Sleep(time.Second * 1) - } - - return errors.Errorf("exceeded wait limit") -} -const waitLimitSec int64 = 300 + if err := restorer.Finalize(ctx); err != nil { + return rr, errors.Wrap(err, "failed to finalize restore") + } -func (r *ReconcilePerconaXtraDBClusterRestore) waitForPodsShutdown(ls map[string]string, namespace string, gracePeriodSec int64) error { - for i := int64(0); i < waitLimitSec+gracePeriodSec; i++ { - pods := corev1.PodList{} + if cr.Spec.PITR != nil { + if cluster.Spec.Pause { + err = k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { + current := new(api.PerconaXtraDBCluster) + err := r.client.Get(ctx, types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, current) + if err != nil { + return errors.Wrap(err, "get cluster") + } + current.Spec.Pause = false + current.Spec.PXC.Size = 1 + current.Spec.AllowUnsafeConfig = true + return r.client.Update(ctx, current) + }) + if err != nil { + return rr, errors.Wrap(err, "update cluster") + } + return rr, nil + } else { + if cluster.Status.ObservedGeneration == cluster.Generation && cluster.Status.PXC.Status == api.AppStateReady { + log.Info("Waiting for cluster to start", "cluster", cluster.Name) + return rr, nil + } + } + + log.Info("point-in-time recovering", "cluster", cr.Spec.PXCCluster) + err = r.pitr(ctx, cr, bcp, cluster) + if err != nil { + if errors.Is(err, errWaitInit) { + return rr, nil + } + return rr, errors.Wrap(err, "run pitr") + } + statusState = api.RestorePITR + return rr, nil + } - err := r.client.List( - context.TODO(), - &pods, - &client.ListOptions{ - Namespace: namespace, - LabelSelector: labels.SelectorFromSet(ls), - }, - ) + log.Info("starting cluster", "cluster", cr.Spec.PXCCluster) + statusState = api.RestoreStartCluster + case api.RestorePITR: + restorer, err := r.getRestorer(cr, bcp, cluster) if err != nil { - return errors.Wrap(err, "get pods list") + return rr, errors.Wrap(err, "failed to get restorer") } - - if len(pods.Items) == 0 { - return nil + restorerJob, err := restorer.PITRJob() + if err != nil { + return rr, errors.Wrap(err, "failed to create restore job") + } + job := new(batchv1.Job) + if err := r.client.Get(ctx, types.NamespacedName{ + Name: restorerJob.Name, + Namespace: restorerJob.Namespace, + }, job); err != nil { + return rr, errors.Wrap(err, "failed to get restore job") } - time.Sleep(time.Second * 1) - } - - return errors.Errorf("exceeded wait limit") -} - -func (r *ReconcilePerconaXtraDBClusterRestore) waitForPVCShutdown(ls map[string]string, namespace string) error { - for i := int64(0); i < waitLimitSec; i++ { - pvcs := corev1.PersistentVolumeClaimList{} - - err := r.client.List( - context.TODO(), - &pvcs, - &client.ListOptions{ - Namespace: namespace, - LabelSelector: labels.SelectorFromSet(ls), - }, - ) + finished, err := isJobFinished(job) if err != nil { - return errors.Wrap(err, "get pvc list") + statusState = api.RestoreFailed + statusMsg = err.Error() + return rr, err } - - if len(pvcs.Items) == 1 { - return nil + if !finished { + log.Info("Waiting for restore job to finish", "job", job.Name) + return rr, nil + } + if err := restorer.Finalize(ctx); err != nil { + return rr, errors.Wrap(err, "failed to finalize restore") } - time.Sleep(time.Second * 1) - } - - return errors.Errorf("exceeded wait limit") -} - -func (r *ReconcilePerconaXtraDBClusterRestore) setStatus(cr *api.PerconaXtraDBClusterRestore, state api.BcpRestoreStates, comments string) error { - cr.Status.State = state - switch state { - case api.RestoreSucceeded: - tm := metav1.NewTime(time.Now()) - cr.Status.CompletedAt = &tm - } - - cr.Status.Comments = comments + log.Info("starting cluster", "cluster", cr.Spec.PXCCluster) + statusState = api.RestoreStartCluster + case api.RestoreStartCluster: + if cluster.Spec.Pause || (cr.Status.PXCSize != 0 && cluster.Spec.PXC.Size != cr.Status.PXCSize) { + err = k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { + current := new(api.PerconaXtraDBCluster) + err := r.client.Get(ctx, types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, current) + if err != nil { + return errors.Wrap(err, "get cluster") + } + current.Spec.Pause = false + if cr.Status.PXCSize != 0 { + current.Spec.PXC.Size = cr.Status.PXCSize + current.Spec.AllowUnsafeConfig = cr.Status.AllowUnsafeConfig + } + return r.client.Update(ctx, current) + }) + if err != nil { + return rr, errors.Wrap(err, "update cluster") + } + } else { + if cluster.Status.ObservedGeneration == cluster.Generation && cluster.Status.PXC.Status == api.AppStateReady { + statusState = api.RestoreSucceeded + return rr, nil + } + } - err := r.client.Status().Update(context.TODO(), cr) - if err != nil { - return errors.Wrap(err, "send update") + log.Info("Waiting for cluster to start", "cluster", cluster.Name) } - return nil + return rr, nil } diff --git a/pkg/controller/pxcrestore/restore.go b/pkg/controller/pxcrestore/restore.go index 232d051337..48e34fe089 100644 --- a/pkg/controller/pxcrestore/restore.go +++ b/pkg/controller/pxcrestore/restore.go @@ -2,22 +2,14 @@ package pxcrestore import ( "context" - "time" "github.com/pkg/errors" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" - "github.com/percona/percona-xtradb-cluster-operator/pkg/k8s" ) func (r *ReconcilePerconaXtraDBClusterRestore) restore(ctx context.Context, cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster) error { - log := logf.FromContext(ctx) - if cluster.Spec.Backup == nil { return errors.New("undefined backup section in a cluster spec") } @@ -30,25 +22,23 @@ func (r *ReconcilePerconaXtraDBClusterRestore) restore(ctx context.Context, cr * if err != nil { return errors.Wrap(err, "failed to get restore job") } - if err = k8s.SetControllerReference(cr, job, r.scheme); err != nil { + if err = controllerutil.SetControllerReference(cr, job, r.scheme); err != nil { return err } if err = restorer.Init(ctx); err != nil { return errors.Wrap(err, "failed to init restore") } - defer func() { - if derr := restorer.Finalize(ctx); derr != nil { - log.Error(derr, "failed to finalize restore") - } - }() - return r.createJob(ctx, job) + err = r.client.Create(ctx, job) + if err != nil { + return errors.Wrap(err, "create job") + } + + return nil } func (r *ReconcilePerconaXtraDBClusterRestore) pitr(ctx context.Context, cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster) error { - log := logf.FromContext(ctx) - restorer, err := r.getRestorer(cr, bcp, cluster) if err != nil { return errors.Wrap(err, "failed to get restorer") @@ -57,19 +47,19 @@ func (r *ReconcilePerconaXtraDBClusterRestore) pitr(ctx context.Context, cr *api if err != nil { return errors.Wrap(err, "failed to create pitr restore job") } - if err := k8s.SetControllerReference(cr, job, r.scheme); err != nil { + if err := controllerutil.SetControllerReference(cr, job, r.scheme); err != nil { return err } if err = restorer.Init(ctx); err != nil { return errors.Wrap(err, "failed to init restore") } - defer func() { - if derr := restorer.Finalize(ctx); derr != nil { - log.Error(derr, "failed to finalize restore") - } - }() - return r.createJob(ctx, job) + err = r.client.Create(ctx, job) + if err != nil { + return errors.Wrap(err, "create job") + } + + return nil } func (r *ReconcilePerconaXtraDBClusterRestore) validate(ctx context.Context, cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster) error { @@ -99,34 +89,3 @@ func (r *ReconcilePerconaXtraDBClusterRestore) validate(ctx context.Context, cr } return nil } - -func (r *ReconcilePerconaXtraDBClusterRestore) createJob(ctx context.Context, job *batchv1.Job) error { - err := r.client.Create(ctx, job) - if err != nil { - return errors.Wrap(err, "create job") - } - - for { - time.Sleep(time.Second * 1) - - checkJob := batchv1.Job{} - err := r.client.Get(ctx, types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, &checkJob) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - return errors.Wrap(err, "get job status") - } - for _, cond := range checkJob.Status.Conditions { - if cond.Status != corev1.ConditionTrue { - continue - } - switch cond.Type { - case batchv1.JobComplete: - return nil - case batchv1.JobFailed: - return errors.New(cond.Message) - } - } - } -} diff --git a/pkg/controller/pxcrestore/restorer.go b/pkg/controller/pxcrestore/restorer.go index 04fa0fb237..8507aa76f0 100644 --- a/pkg/controller/pxcrestore/restorer.go +++ b/pkg/controller/pxcrestore/restorer.go @@ -4,7 +4,6 @@ import ( "context" "sort" "strings" - "time" "github.com/pkg/errors" batchv1 "k8s.io/api/batch/v1" @@ -13,13 +12,18 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" - "github.com/percona/percona-xtradb-cluster-operator/pkg/k8s" "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup" "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup/storage" ) +var ( + errWaitValidate = errors.New("wait for validation") + errWaitInit = errors.New("wait for init") +) + type Restorer interface { Init(ctx context.Context) error Job() (*batchv1.Job, error) @@ -84,36 +88,38 @@ func (s *pvc) Validate(ctx context.Context) error { if err != nil { return errors.Wrap(err, "restore pod") } - if err := k8s.SetControllerReference(s.cr, pod, s.scheme); err != nil { + if err := controllerutil.SetControllerReference(s.cr, pod, s.scheme); err != nil { return err } pod.Name += "-verify" pod.Spec.Containers[0].Command = []string{"bash", "-c", `[[ $(stat -c%s /backup/xtrabackup.stream) -gt 5000000 ]]`} pod.Spec.RestartPolicy = corev1.RestartPolicyNever - if err := s.k8sClient.Delete(ctx, pod); client.IgnoreNotFound(err) != nil { - return errors.Wrap(err, "failed to delete") - } - - if err := s.k8sClient.Create(ctx, pod); err != nil { - return errors.Wrap(err, "failed to create pod") + err = s.k8sClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) + if err != nil { + if k8serrors.IsNotFound(err) { + if err := s.k8sClient.Create(ctx, pod); err != nil { + return errors.Wrap(err, "failed to create pod") + } + return errWaitValidate + } + return errors.Wrap(err, "get pod status") } - for { - time.Sleep(time.Second * 1) - err := s.k8sClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) - if err != nil { - return errors.Wrap(err, "get pod status") + switch pod.Status.Phase { + case corev1.PodFailed: + if err := s.k8sClient.Delete(ctx, pod); err != nil { + return errors.Wrap(err, "failed to delete") } - if pod.Status.Phase == corev1.PodFailed { - return errors.Errorf("backup files not found on %s", destination) - } - if pod.Status.Phase == corev1.PodSucceeded { - break + return errors.Errorf("backup files not found on %s", destination) + case corev1.PodSucceeded: + if err := s.k8sClient.Delete(ctx, pod); err != nil { + return errors.Wrap(err, "failed to delete") } + return nil + default: + return errWaitValidate } - - return nil } func (s *pvc) Job() (*batchv1.Job, error) { @@ -128,55 +134,65 @@ func (s *pvc) Init(ctx context.Context) error { destination := s.bcp.Status.Destination svc := backup.PVCRestoreService(s.cr) - if err := k8s.SetControllerReference(s.cr, svc, s.scheme); err != nil { + if err := controllerutil.SetControllerReference(s.cr, svc, s.scheme); err != nil { return err } pod, err := backup.PVCRestorePod(s.cr, s.bcp.Status.StorageName, destination.BackupName(), s.cluster) if err != nil { return errors.Wrap(err, "restore pod") } - if err := k8s.SetControllerReference(s.cr, pod, s.scheme); err != nil { - return err - } - if err := s.k8sClient.Delete(ctx, svc); client.IgnoreNotFound(err) != nil { - return err - } - if err := s.k8sClient.Delete(ctx, pod); client.IgnoreNotFound(err) != nil { + if err := controllerutil.SetControllerReference(s.cr, pod, s.scheme); err != nil { return err } - err = s.k8sClient.Create(ctx, svc) - if err != nil { - return errors.Wrap(err, "create service") + initInProcess := true + + if err := s.k8sClient.Get(ctx, types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, svc); err != nil { + if k8serrors.IsNotFound(err) { + initInProcess = false + } } - err = s.k8sClient.Create(ctx, pod) - if err != nil { - return errors.Wrap(err, "create pod") + + if err := s.k8sClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: svc.Namespace}, pod); err != nil { + if k8serrors.IsNotFound(err) { + initInProcess = false + } } - for { - time.Sleep(time.Second * 1) - err := s.k8sClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) + if !initInProcess { + if err := s.k8sClient.Delete(ctx, svc); client.IgnoreNotFound(err) != nil { + return err + } + if err := s.k8sClient.Delete(ctx, pod); client.IgnoreNotFound(err) != nil { + return err + } + + err = s.k8sClient.Create(ctx, svc) if err != nil { - return errors.Wrap(err, "get pod status") + return errors.Wrap(err, "create service") } - if pod.Status.Phase == corev1.PodRunning { - break + err = s.k8sClient.Create(ctx, pod) + if err != nil { + return errors.Wrap(err, "create pod") } } + + if pod.Status.Phase != corev1.PodRunning { + return errWaitInit + } return nil } func (s *pvc) Finalize(ctx context.Context) error { svc := backup.PVCRestoreService(s.cr) - if err := s.k8sClient.Delete(ctx, svc); err != nil { + if err := s.k8sClient.Delete(ctx, svc); client.IgnoreNotFound(err) != nil { return errors.Wrap(err, "failed to delete pvc service") } pod, err := backup.PVCRestorePod(s.cr, s.bcp.Status.StorageName, s.bcp.Status.Destination.BackupName(), s.cluster) if err != nil { return err } - if err := s.k8sClient.Delete(ctx, pod); err != nil { + if err := s.k8sClient.Delete(ctx, pod); client.IgnoreNotFound(err) != nil { return errors.Wrap(err, "failed to delete pvc pod") } return nil diff --git a/pkg/controller/pxcrestore/util.go b/pkg/controller/pxcrestore/util.go new file mode 100644 index 0000000000..b7e2d32dee --- /dev/null +++ b/pkg/controller/pxcrestore/util.go @@ -0,0 +1,210 @@ +package pxcrestore + +import ( + "context" + "strings" + "time" + + "github.com/pkg/errors" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" + "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app" + "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset" +) + +var ( + errWaitingPods = errors.New("waiting for pods to be deleted") + errWaitingPVC = errors.New("waiting for pvc to be deleted") +) + +func getBackup(ctx context.Context, cl client.Client, cr *api.PerconaXtraDBClusterRestore) (*api.PerconaXtraDBClusterBackup, error) { + if cr.Spec.BackupSource != nil { + status := cr.Spec.BackupSource.DeepCopy() + status.State = api.BackupSucceeded + status.CompletedAt = nil + status.LastScheduled = nil + return &api.PerconaXtraDBClusterBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name, + Namespace: cr.Namespace, + }, + Spec: api.PXCBackupSpec{ + PXCCluster: cr.Spec.PXCCluster, + StorageName: cr.Spec.BackupSource.StorageName, + }, + Status: *status, + }, nil + } + + bcp := &api.PerconaXtraDBClusterBackup{} + err := cl.Get(ctx, types.NamespacedName{Name: cr.Spec.BackupName, Namespace: cr.Namespace}, bcp) + if err != nil { + err = errors.Wrapf(err, "get backup %s", cr.Spec.BackupName) + return bcp, err + } + if bcp.Status.State != api.BackupSucceeded { + err = errors.Errorf("backup %s didn't finished yet, current state: %s", bcp.Name, bcp.Status.State) + return bcp, err + } + + return bcp, nil +} + +func stopCluster(ctx context.Context, cl client.Client, c *api.PerconaXtraDBCluster) error { + patch := client.MergeFrom(c.DeepCopy()) + c.Spec.Pause = true + err := cl.Patch(ctx, c, patch) + if err != nil { + return errors.Wrap(err, "shutdown pods") + } + + ls := statefulset.NewNode(c).Labels() + stopped, err := isClusterStopped(ctx, cl, ls, c.Namespace) + if err != nil { + return errors.Wrap(err, "check cluster state") + } + if !stopped { + return errWaitingPods + } + + pvcs := corev1.PersistentVolumeClaimList{} + err = cl.List( + ctx, + &pvcs, + &client.ListOptions{ + Namespace: c.Namespace, + LabelSelector: labels.SelectorFromSet(ls), + }, + ) + if err != nil { + return errors.Wrap(err, "get pvc list") + } + + pxcNode := statefulset.NewNode(c) + pvcNameTemplate := app.DataVolumeName + "-" + pxcNode.StatefulSet().Name + for _, pvc := range pvcs.Items { + // check prefix just in case, to be sure we're not going to delete a wrong pvc + if pvc.Name == pvcNameTemplate+"-0" || !strings.HasPrefix(pvc.Name, pvcNameTemplate) { + continue + } + + err = cl.Delete(ctx, &pvc) + if err != nil { + return errors.Wrap(err, "delete pvc") + } + } + + pvcDeleted, err := isPVCDeleted(ctx, cl, ls, c.Namespace) + if err != nil { + return errors.Wrap(err, "check pvc state") + } + + if !pvcDeleted { + return errWaitingPVC + } + + return nil +} + +func setStatus(ctx context.Context, cl client.Client, cr *api.PerconaXtraDBClusterRestore, state api.BcpRestoreStates, comments string) error { + cr.Status.State = state + switch state { + case api.RestoreSucceeded: + tm := metav1.NewTime(time.Now()) + cr.Status.CompletedAt = &tm + } + + cr.Status.Comments = comments + + err := cl.Status().Update(ctx, cr) + if err != nil { + return errors.Wrap(err, "send update") + } + + return nil +} + +func isOtherRestoreInProgress(ctx context.Context, cl client.Client, cr *api.PerconaXtraDBClusterRestore) (*api.PerconaXtraDBClusterRestore, error) { + rJobsList := &api.PerconaXtraDBClusterRestoreList{} + err := cl.List( + ctx, + rJobsList, + &client.ListOptions{ + Namespace: cr.Namespace, + }, + ) + if err != nil { + return nil, errors.Wrap(err, "get restore jobs list") + } + + for _, j := range rJobsList.Items { + if j.Spec.PXCCluster == cr.Spec.PXCCluster && + j.Name != cr.Name && j.Status.State != api.RestoreFailed && + j.Status.State != api.RestoreSucceeded { + return &j, nil + } + } + return nil, nil +} + +func isClusterStopped(ctx context.Context, cl client.Client, ls map[string]string, namespace string) (bool, error) { + pods := corev1.PodList{} + + err := cl.List( + ctx, + &pods, + &client.ListOptions{ + Namespace: namespace, + LabelSelector: labels.SelectorFromSet(ls), + }, + ) + if err != nil { + return false, errors.Wrap(err, "get pods list") + } + + return len(pods.Items) == 0, nil +} + +func isPVCDeleted(ctx context.Context, cl client.Client, ls map[string]string, namespace string) (bool, error) { + pvcs := corev1.PersistentVolumeClaimList{} + + err := cl.List( + ctx, + &pvcs, + &client.ListOptions{ + Namespace: namespace, + LabelSelector: labels.SelectorFromSet(ls), + }, + ) + if err != nil { + return false, errors.Wrap(err, "get pvc list") + } + + if len(pvcs.Items) == 1 { + return true, nil + } + + return false, nil +} + +func isJobFinished(checkJob *batchv1.Job) (bool, error) { + for _, c := range checkJob.Status.Conditions { + if c.Status != corev1.ConditionTrue { + continue + } + + switch c.Type { + case batchv1.JobComplete: + return true, nil + case batchv1.JobFailed: + return false, errors.Errorf("job %s failed: %s", checkJob.Name, c.Message) + } + } + return false, nil +} diff --git a/pkg/k8s/setowner.go b/pkg/k8s/setowner.go index afa285abfe..16baff4d4b 100644 --- a/pkg/k8s/setowner.go +++ b/pkg/k8s/setowner.go @@ -7,16 +7,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) -// SetControllerReference sets owner as a owner for the object obj -func SetControllerReference(owner runtime.Object, obj metav1.Object, scheme *runtime.Scheme) error { - ownerRef, err := OwnerRef(owner, scheme) - if err != nil { - return err - } - obj.SetOwnerReferences(append(obj.GetOwnerReferences(), ownerRef)) - return nil -} - // OwnerRef returns OwnerReference to object func OwnerRef(ro runtime.Object, scheme *runtime.Scheme) (metav1.OwnerReference, error) { gvk, err := apiutil.GVKForObject(ro, scheme) From d0609dabda5d9db1f049ef24fa32655589741105 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Tue, 30 Jan 2024 15:03:52 +0200 Subject: [PATCH 2/9] fix unit test --- pkg/controller/pxcrestore/restore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/pxcrestore/restore_test.go b/pkg/controller/pxcrestore/restore_test.go index 44e810edda..c0a0139ffd 100644 --- a/pkg/controller/pxcrestore/restore_test.go +++ b/pkg/controller/pxcrestore/restore_test.go @@ -266,7 +266,7 @@ func TestValidate(t *testing.T) { r := reconciler(cl) r.newStorageClientFunc = tt.fakeStorageClientFunc - bcp, err := r.getBackup(ctx, tt.cr) + bcp, err := getBackup(ctx, cl, tt.cr) if err != nil { t.Fatal(err) } From d3d247b75bcdcfe0248babca59ec52bc7a5507f1 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 31 Jan 2024 14:19:01 +0200 Subject: [PATCH 3/9] fix for pvc restores --- pkg/controller/pxcrestore/controller.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/controller/pxcrestore/controller.go b/pkg/controller/pxcrestore/controller.go index dbfe8ff262..bc66fccd56 100644 --- a/pkg/controller/pxcrestore/controller.go +++ b/pkg/controller/pxcrestore/controller.go @@ -235,10 +235,6 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re return rr, nil } - if err := restorer.Finalize(ctx); err != nil { - return rr, errors.Wrap(err, "failed to finalize restore") - } - if cr.Spec.PITR != nil { if cluster.Spec.Pause { err = k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { @@ -304,9 +300,6 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re log.Info("Waiting for restore job to finish", "job", job.Name) return rr, nil } - if err := restorer.Finalize(ctx); err != nil { - return rr, errors.Wrap(err, "failed to finalize restore") - } log.Info("starting cluster", "cluster", cr.Spec.PXCCluster) statusState = api.RestoreStartCluster @@ -330,6 +323,14 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re } } else { if cluster.Status.ObservedGeneration == cluster.Generation && cluster.Status.PXC.Status == api.AppStateReady { + restorer, err := r.getRestorer(cr, bcp, cluster) + if err != nil { + return rr, errors.Wrap(err, "failed to get restorer") + } + if err := restorer.Finalize(ctx); err != nil { + return rr, errors.Wrap(err, "failed to finalize restore") + } + statusState = api.RestoreSucceeded return rr, nil } From 05af3823128c47b07779cf338bfb1302bf24533a Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 1 Feb 2024 00:47:59 +0200 Subject: [PATCH 4/9] refactor --- pkg/controller/pxcrestore/controller.go | 72 +++++++++++---- .../{restore_test.go => controller_test.go} | 6 +- pkg/controller/pxcrestore/helpers_test.go | 5 + pkg/controller/pxcrestore/restore.go | 91 ------------------- pkg/controller/pxcrestore/restorer.go | 10 +- pkg/pxc/backup/restore.go | 10 +- 6 files changed, 79 insertions(+), 115 deletions(-) rename pkg/controller/pxcrestore/{restore_test.go => controller_test.go} (98%) delete mode 100644 pkg/controller/pxcrestore/restore.go diff --git a/pkg/controller/pxcrestore/controller.go b/pkg/controller/pxcrestore/controller.go index bc66fccd56..b15ef75b1f 100644 --- a/pkg/controller/pxcrestore/controller.go +++ b/pkg/controller/pxcrestore/controller.go @@ -164,6 +164,11 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re return rr, errors.Wrap(err, "get backup") } + restorer, err := r.getRestorer(cr, bcp, cluster) + if err != nil { + return rr, errors.Wrap(err, "failed to get restorer") + } + switch statusState { case api.RestoreNew: annotations := cr.GetAnnotations() @@ -174,7 +179,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re statusMsg = fmt.Sprintf("Backup doesn't guarantee consistent recovery with PITR. Annotate PerconaXtraDBClusterRestore with %s to force it.", api.AnnotationUnsafePITR) return rr, nil } - err = r.validate(ctx, cr, bcp, cluster) + err = validate(ctx, restorer, cr) if err != nil { if errors.Is(err, errWaitValidate) { return rr, nil @@ -198,8 +203,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re } log.Info("starting restore", "cluster", cr.Spec.PXCCluster, "backup", cr.Spec.BackupName) - err = r.restore(ctx, cr, bcp, cluster) - if err != nil { + if err := createRestoreJob(ctx, r.client, restorer, false); err != nil { if errors.Is(err, errWaitInit) { return rr, nil } @@ -208,10 +212,6 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re } statusState = api.RestoreRestore case api.RestoreRestore: - restorer, err := r.getRestorer(cr, bcp, cluster) - if err != nil { - return rr, errors.Wrap(err, "failed to get restorer") - } restorerJob, err := restorer.Job() if err != nil { return rr, errors.Wrap(err, "failed to create restore job") @@ -260,8 +260,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re } log.Info("point-in-time recovering", "cluster", cr.Spec.PXCCluster) - err = r.pitr(ctx, cr, bcp, cluster) - if err != nil { + if err := createRestoreJob(ctx, r.client, restorer, true); err != nil { if errors.Is(err, errWaitInit) { return rr, nil } @@ -274,10 +273,6 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re log.Info("starting cluster", "cluster", cr.Spec.PXCCluster) statusState = api.RestoreStartCluster case api.RestorePITR: - restorer, err := r.getRestorer(cr, bcp, cluster) - if err != nil { - return rr, errors.Wrap(err, "failed to get restorer") - } restorerJob, err := restorer.PITRJob() if err != nil { return rr, errors.Wrap(err, "failed to create restore job") @@ -323,10 +318,6 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re } } else { if cluster.Status.ObservedGeneration == cluster.Generation && cluster.Status.PXC.Status == api.AppStateReady { - restorer, err := r.getRestorer(cr, bcp, cluster) - if err != nil { - return rr, errors.Wrap(err, "failed to get restorer") - } if err := restorer.Finalize(ctx); err != nil { return rr, errors.Wrap(err, "failed to finalize restore") } @@ -341,3 +332,50 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re return rr, nil } + +func createRestoreJob(ctx context.Context, cl client.Client, restorer Restorer, pitr bool) error { + if err := restorer.Init(ctx); err != nil { + return errors.Wrap(err, "failed to init restore") + } + + job, err := restorer.Job() + if err != nil { + return errors.Wrap(err, "failed to get restore job") + } + if pitr { + job, err = restorer.PITRJob() + if err != nil { + return errors.Wrap(err, "failed to create pitr restore job") + } + } + + if err := cl.Create(ctx, job); err != nil { + return errors.Wrap(err, "create job") + } + + return nil +} + +func validate(ctx context.Context, restorer Restorer, cr *api.PerconaXtraDBClusterRestore) error { + job, err := restorer.Job() + if err != nil { + return errors.Wrap(err, "failed to create restore job") + } + if err := restorer.ValidateJob(ctx, job); err != nil { + return errors.Wrap(err, "failed to validate job") + } + + if cr.Spec.PITR != nil { + job, err := restorer.PITRJob() + if err != nil { + return errors.Wrap(err, "failed to create pitr restore job") + } + if err := restorer.ValidateJob(ctx, job); err != nil { + return errors.Wrap(err, "failed to validate job") + } + } + if err := restorer.Validate(ctx); err != nil { + return errors.Wrap(err, "failed to validate backup existence") + } + return nil +} diff --git a/pkg/controller/pxcrestore/restore_test.go b/pkg/controller/pxcrestore/controller_test.go similarity index 98% rename from pkg/controller/pxcrestore/restore_test.go rename to pkg/controller/pxcrestore/controller_test.go index c0a0139ffd..1dcd1fa3b4 100644 --- a/pkg/controller/pxcrestore/restore_test.go +++ b/pkg/controller/pxcrestore/controller_test.go @@ -270,7 +270,11 @@ func TestValidate(t *testing.T) { if err != nil { t.Fatal(err) } - err = r.validate(ctx, tt.cr, bcp, tt.cluster) + restorer, err := r.getRestorer(tt.cr, bcp, tt.cluster) + if err != nil { + t.Fatal(err) + } + err = validate(ctx, restorer, tt.cr) errStr := "" if err != nil { errStr = err.Error() diff --git a/pkg/controller/pxcrestore/helpers_test.go b/pkg/controller/pxcrestore/helpers_test.go index 3976c3d61f..e1120ff5cd 100644 --- a/pkg/controller/pxcrestore/helpers_test.go +++ b/pkg/controller/pxcrestore/helpers_test.go @@ -14,6 +14,7 @@ import ( api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" fakestorage "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup/storage/fake" + "github.com/percona/percona-xtradb-cluster-operator/version" ) func readDefaultCR(t *testing.T, name, namespace string) *api.PerconaXtraDBCluster { @@ -138,6 +139,9 @@ func reconciler(cl client.Client) *ReconcilePerconaXtraDBClusterRestore { client: cl, scheme: cl.Scheme(), newStorageClientFunc: fakestorage.NewFakeClient, + serverVersion: &version.ServerVersion{ + Platform: version.PlatformKubernetes, + }, } } @@ -146,6 +150,7 @@ func buildFakeClient(objs ...runtime.Object) client.Client { s := scheme.Scheme s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBClusterRestore)) + s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBClusterRestoreList)) s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBClusterBackup)) s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBCluster)) diff --git a/pkg/controller/pxcrestore/restore.go b/pkg/controller/pxcrestore/restore.go deleted file mode 100644 index 48e34fe089..0000000000 --- a/pkg/controller/pxcrestore/restore.go +++ /dev/null @@ -1,91 +0,0 @@ -package pxcrestore - -import ( - "context" - - "github.com/pkg/errors" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" -) - -func (r *ReconcilePerconaXtraDBClusterRestore) restore(ctx context.Context, cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster) error { - if cluster.Spec.Backup == nil { - return errors.New("undefined backup section in a cluster spec") - } - - restorer, err := r.getRestorer(cr, bcp, cluster) - if err != nil { - return errors.Wrap(err, "failed to get restorer") - } - job, err := restorer.Job() - if err != nil { - return errors.Wrap(err, "failed to get restore job") - } - if err = controllerutil.SetControllerReference(cr, job, r.scheme); err != nil { - return err - } - - if err = restorer.Init(ctx); err != nil { - return errors.Wrap(err, "failed to init restore") - } - - err = r.client.Create(ctx, job) - if err != nil { - return errors.Wrap(err, "create job") - } - - return nil -} - -func (r *ReconcilePerconaXtraDBClusterRestore) pitr(ctx context.Context, cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster) error { - restorer, err := r.getRestorer(cr, bcp, cluster) - if err != nil { - return errors.Wrap(err, "failed to get restorer") - } - job, err := restorer.PITRJob() - if err != nil { - return errors.Wrap(err, "failed to create pitr restore job") - } - if err := controllerutil.SetControllerReference(cr, job, r.scheme); err != nil { - return err - } - if err = restorer.Init(ctx); err != nil { - return errors.Wrap(err, "failed to init restore") - } - - err = r.client.Create(ctx, job) - if err != nil { - return errors.Wrap(err, "create job") - } - - return nil -} - -func (r *ReconcilePerconaXtraDBClusterRestore) validate(ctx context.Context, cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster) error { - restorer, err := r.getRestorer(cr, bcp, cluster) - if err != nil { - return errors.Wrap(err, "failed to get restorer") - } - job, err := restorer.Job() - if err != nil { - return errors.Wrap(err, "failed to create restore job") - } - if err := restorer.ValidateJob(ctx, job); err != nil { - return errors.Wrap(err, "failed to validate job") - } - - if cr.Spec.PITR != nil { - job, err := restorer.PITRJob() - if err != nil { - return errors.Wrap(err, "failed to create pitr restore job") - } - if err := restorer.ValidateJob(ctx, job); err != nil { - return errors.Wrap(err, "failed to validate job") - } - } - if err := restorer.Validate(ctx); err != nil { - return errors.Wrap(err, "failed to validate backup existence") - } - return nil -} diff --git a/pkg/controller/pxcrestore/restorer.go b/pkg/controller/pxcrestore/restorer.go index 8507aa76f0..9065379867 100644 --- a/pkg/controller/pxcrestore/restorer.go +++ b/pkg/controller/pxcrestore/restorer.go @@ -39,11 +39,11 @@ func (s *s3) Init(context.Context) error { return nil } func (s *s3) Finalize(context.Context) error { return nil } func (s *s3) Job() (*batchv1.Job, error) { - return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, false) + return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, s.scheme, false) } func (s *s3) PITRJob() (*batchv1.Job, error) { - return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, true) + return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, s.scheme, true) } func (s *s3) ValidateJob(ctx context.Context, job *batchv1.Job) error { @@ -123,7 +123,7 @@ func (s *pvc) Validate(ctx context.Context) error { } func (s *pvc) Job() (*batchv1.Job, error) { - return backup.RestoreJob(s.cr, s.bcp, s.cluster, "", false) + return backup.RestoreJob(s.cr, s.bcp, s.cluster, "", s.scheme, false) } func (s *pvc) PITRJob() (*batchv1.Job, error) { @@ -204,11 +204,11 @@ func (s *azure) Init(context.Context) error { return nil } func (s *azure) Finalize(context.Context) error { return nil } func (s *azure) Job() (*batchv1.Job, error) { - return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, false) + return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, s.scheme, false) } func (s *azure) PITRJob() (*batchv1.Job, error) { - return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, true) + return backup.RestoreJob(s.cr, s.bcp, s.cluster, s.bcp.Status.Destination, s.scheme, true) } func (s *azure) Validate(ctx context.Context) error { diff --git a/pkg/pxc/backup/restore.go b/pkg/pxc/backup/restore.go index b29053f775..a9fc36511e 100644 --- a/pkg/pxc/backup/restore.go +++ b/pkg/pxc/backup/restore.go @@ -10,6 +10,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" @@ -136,7 +138,7 @@ func PVCRestorePod(cr *api.PerconaXtraDBClusterRestore, bcpStorageName, pvcName }, nil } -func RestoreJob(cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster, destination api.PXCBackupDestination, pitr bool) (*batchv1.Job, error) { +func RestoreJob(cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClusterBackup, cluster *api.PerconaXtraDBCluster, destination api.PXCBackupDestination, scheme *runtime.Scheme, pitr bool) (*batchv1.Job, error) { switch bcp.Status.GetStorageType(cluster) { case api.BackupStorageAzure: if bcp.Status.Azure == nil { @@ -251,6 +253,12 @@ func RestoreJob(cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClust BackoffLimit: func(i int32) *int32 { return &i }(4), }, } + if err := controllerutil.SetControllerReference(cr, job, scheme); err != nil { + return nil, errors.Wrap(err, "set controller reference") + } + for _, or := range job.OwnerReferences { + or.BlockOwnerDeletion = nil + } return job, nil } From 136bcc5e29b9344184fdd98196e9c6c7c85bfdcb Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 1 Feb 2024 12:10:16 +0200 Subject: [PATCH 5/9] fix tests --- pkg/pxc/backup/restore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pxc/backup/restore.go b/pkg/pxc/backup/restore.go index a9fc36511e..188dbfe555 100644 --- a/pkg/pxc/backup/restore.go +++ b/pkg/pxc/backup/restore.go @@ -256,8 +256,8 @@ func RestoreJob(cr *api.PerconaXtraDBClusterRestore, bcp *api.PerconaXtraDBClust if err := controllerutil.SetControllerReference(cr, job, scheme); err != nil { return nil, errors.Wrap(err, "set controller reference") } - for _, or := range job.OwnerReferences { - or.BlockOwnerDeletion = nil + for i := range job.OwnerReferences { + job.OwnerReferences[i].BlockOwnerDeletion = nil } return job, nil } From 8f512a4d45389d215c6d3c9b758339d390f33fdf Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Fri, 2 Feb 2024 01:28:06 +0200 Subject: [PATCH 6/9] fix `security-context` test --- .../compare/pod_restore-src-restore-pvc-sec-context-oc.yml | 3 ++- .../compare/pod_restore-src-restore-pvc-sec-context.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context-oc.yml b/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context-oc.yml index f794041b4a..92dcf141f6 100644 --- a/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context-oc.yml +++ b/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context-oc.yml @@ -7,7 +7,8 @@ metadata: name: restore-src-restore-pvc-sec-context name: restore-src-restore-pvc-sec-context ownerReferences: - - controller: true + - blockOwnerDeletion: true + controller: true kind: PerconaXtraDBClusterRestore name: restore-pvc spec: diff --git a/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context.yml b/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context.yml index 52b8204712..f830ea70e0 100644 --- a/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context.yml +++ b/e2e-tests/security-context/compare/pod_restore-src-restore-pvc-sec-context.yml @@ -7,7 +7,8 @@ metadata: name: restore-src-restore-pvc-sec-context name: restore-src-restore-pvc-sec-context ownerReferences: - - controller: true + - blockOwnerDeletion: true + controller: true kind: PerconaXtraDBClusterRestore name: restore-pvc spec: From 34666c13c06e04868d28383db9c5f7c802ce9a46 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 5 Feb 2024 15:55:34 +0200 Subject: [PATCH 7/9] add unit-test --- pkg/apis/pxc/v1/pxc_prestore_types.go | 1 - pkg/controller/pxc/upgrade.go | 2 +- pkg/controller/pxcrestore/controller.go | 4 +- pkg/controller/pxcrestore/controller_test.go | 217 ++++++++++++++++++- pkg/controller/pxcrestore/helpers_test.go | 25 ++- pkg/controller/pxcrestore/restorer.go | 14 +- 6 files changed, 245 insertions(+), 18 deletions(-) diff --git a/pkg/apis/pxc/v1/pxc_prestore_types.go b/pkg/apis/pxc/v1/pxc_prestore_types.go index c82e482d31..68270c3b52 100644 --- a/pkg/apis/pxc/v1/pxc_prestore_types.go +++ b/pkg/apis/pxc/v1/pxc_prestore_types.go @@ -65,7 +65,6 @@ type BcpRestoreStates string const ( RestoreNew BcpRestoreStates = "" - RestoreStarting BcpRestoreStates = "Starting" RestoreStopCluster BcpRestoreStates = "Stopping Cluster" RestoreRestore BcpRestoreStates = "Restoring" RestoreStartCluster BcpRestoreStates = "Starting Cluster" diff --git a/pkg/controller/pxc/upgrade.go b/pkg/controller/pxc/upgrade.go index 8f7c369f2f..93c25f751b 100644 --- a/pkg/controller/pxc/upgrade.go +++ b/pkg/controller/pxc/upgrade.go @@ -666,7 +666,7 @@ func (r *ReconcilePerconaXtraDBCluster) isRestoreRunning(clusterName, namespace } switch v.Status.State { - case api.RestoreStarting, api.RestoreStopCluster, api.RestoreRestore, + case api.RestoreStopCluster, api.RestoreRestore, api.RestoreStartCluster, api.RestorePITR: return true, nil } diff --git a/pkg/controller/pxcrestore/controller.go b/pkg/controller/pxcrestore/controller.go index b15ef75b1f..63a4a3586e 100644 --- a/pkg/controller/pxcrestore/controller.go +++ b/pkg/controller/pxcrestore/controller.go @@ -282,7 +282,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re Name: restorerJob.Name, Namespace: restorerJob.Namespace, }, job); err != nil { - return rr, errors.Wrap(err, "failed to get restore job") + return rr, errors.Wrap(err, "failed to get pitr job") } finished, err := isJobFinished(job) @@ -349,7 +349,7 @@ func createRestoreJob(ctx context.Context, cl client.Client, restorer Restorer, } } - if err := cl.Create(ctx, job); err != nil { + if err := cl.Create(ctx, job); err != nil && !k8serrors.IsAlreadyExists(err) { return errors.Wrap(err, "create job") } diff --git a/pkg/controller/pxcrestore/controller_test.go b/pkg/controller/pxcrestore/controller_test.go index 1dcd1fa3b4..4245def7e0 100644 --- a/pkg/controller/pxcrestore/controller_test.go +++ b/pkg/controller/pxcrestore/controller_test.go @@ -4,14 +4,21 @@ import ( "context" "testing" + "github.com/pkg/errors" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" + "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset" + "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup" "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup/storage" fakestorage "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup/storage/fake" "github.com/percona/percona-xtradb-cluster-operator/version" - "github.com/pkg/errors" ) func TestValidate(t *testing.T) { @@ -301,3 +308,211 @@ func (c *fakeStorageClient) ListObjects(_ context.Context, _ string) ([]string, } return []string{"some-dest/backup1", "some-dest/backup2"}, nil } + +// TestOperatorRestart checks that the operator can catch up with the restore process after a restart. +// This test is run for each restore state. It runs reconcile twice for each state. Each reconcile operator should change the restore state. +// This test helps to eliminate errors such as creating an existing Pod without handling the AlreadyExists error. +func TestOperatorRestart(t *testing.T) { + ctx := context.Background() + + const clusterName = "test-cluster" + const namespace = "namespace" + const backupName = clusterName + "-backup" + const restoreName = clusterName + "-restore" + const s3SecretName = "my-cluster-name-backup-s3" + const azureSecretName = "my-cluster-name-backup-azure" + + states := []api.BcpRestoreStates{ + api.RestoreNew, + api.RestoreStopCluster, + api.RestoreRestore, + api.RestoreStartCluster, + api.RestorePITR, + } + + bcp := readDefaultBackup(t, backupName, namespace) + crSecret := readDefaultCRSecret(t, clusterName+"-secrets", namespace) + cluster := readDefaultCR(t, clusterName, namespace) + cluster.Status.PXC.Status = api.AppStateReady + cr := readDefaultRestore(t, restoreName, namespace) + cr.Spec.BackupName = backupName + cr.Spec.PXCCluster = clusterName + + tests := []struct { + name string + bcp *api.PerconaXtraDBClusterBackup + objects []runtime.Object + }{ + { + name: "s3", + bcp: updateResource(bcp, func(bcp *api.PerconaXtraDBClusterBackup) { + bcp.Status.State = api.BackupSucceeded + bcp.Status.Destination.SetS3Destination("some-dest", "dest") + bcp.Spec.StorageName = "s3-us-west" + bcp.Status.S3 = &api.BackupStorageS3Spec{ + Bucket: "some-bucket", + CredentialsSecret: s3SecretName, + } + }), + objects: []runtime.Object{readDefaultS3Secret(t, s3SecretName, namespace)}, + }, + { + name: "azure", + bcp: updateResource(bcp, func(bcp *api.PerconaXtraDBClusterBackup) { + bcp.Status.State = api.BackupSucceeded + bcp.Status.Destination.SetAzureDestination("some-dest", "dest") + bcp.Spec.StorageName = "azure-blob" + bcp.Status.Azure = &api.BackupStorageAzureSpec{ + ContainerPath: "some-bucket", + CredentialsSecret: azureSecretName, + } + }), + objects: []runtime.Object{ + updateResource(readDefaultS3Secret(t, azureSecretName, namespace), func(secret *corev1.Secret) { + secret.Data = map[string][]byte{ + "AZURE_STORAGE_ACCOUNT_NAME": []byte("some-account"), + "AZURE_STORAGE_ACCOUNT_KEY": []byte("some-key"), + } + }), + }, + }, + { + name: "pvc", + bcp: updateResource(bcp, func(bcp *api.PerconaXtraDBClusterBackup) { + bcp.Status.State = api.BackupSucceeded + bcp.Status.Destination.SetPVCDestination("some-dest") + bcp.Status.StorageType = api.BackupStorageFilesystem + }), + objects: []runtime.Object{ + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "restore-src-" + cr.Name + "-" + cr.Spec.PXCCluster, + Namespace: cr.Namespace, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "restore-src-" + cr.Name + "-" + cr.Spec.PXCCluster + "-verify", + Namespace: cr.Namespace, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, + }, + }, + backup.PVCRestoreService(cr), + }, + }, + } + + for _, tt := range tests { + for _, state := range states { + if tt.bcp.Status.StorageType == api.BackupStorageFilesystem && state == api.RestorePITR { + continue + } + t.Run(tt.name+" state "+string(state), func(t *testing.T) { + cr := cr.DeepCopy() + cluster := cluster.DeepCopy() + if state == api.RestorePITR { + cr.Spec.PITR = &api.PITR{ + BackupSource: &api.PXCBackupStatus{ + StorageName: tt.bcp.Spec.StorageName, + }, + } + } + cr.Status.State = state + objects := append(tt.objects, tt.bcp, cr, cluster, crSecret, &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc1", + Namespace: namespace, + Labels: statefulset.NewNode(cluster).Labels(), + }, + }) + cl := buildFakeClient(objects...) + + r := reconciler(cl) + r.newStorageClientFunc = func(ctx context.Context, opts storage.Options) (storage.Storage, error) { + defaultFakeClient, err := fakestorage.NewFakeClient(ctx, opts) + if err != nil { + return nil, err + } + return &fakeStorageClient{defaultFakeClient, false, false}, nil + } + + if state == api.RestoreRestore || state == api.RestorePITR { + restorer, err := r.getRestorer(cr, tt.bcp, cluster) + if err != nil { + t.Fatal(err) + } + job, err := restorer.Job() + if err != nil { + t.Fatal(err) + } + if state == api.RestorePITR { + job, err = restorer.PITRJob() + if err != nil { + t.Fatal(err) + } + } + job.Status.Conditions = []batchv1.JobCondition{ + { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + } + if err := r.client.Create(ctx, job); err != nil { + t.Fatal(err) + } + } + + nn := types.NamespacedName{ + Name: cr.Name, + Namespace: cr.Namespace, + } + req := reconcile.Request{ + NamespacedName: nn, + } + + _, err := r.Reconcile(ctx, req) + if err != nil { + t.Fatal(err) + } + + restore := new(api.PerconaXtraDBClusterRestore) + if err := r.client.Get(ctx, nn, restore); err != nil { + t.Fatal(err) + } + if restore.Status.State == state { + t.Fatal("state not changed") + } + + // Assuming that the operator restarted just before the status update + restore.Status.State = state + if err := r.client.Status().Update(ctx, restore); err != nil { + t.Fatal(err) + } + _, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatal(err) + } + + if err := r.client.Get(ctx, nn, restore); err != nil { + t.Fatal(err) + } + if restore.Status.State == state { + t.Fatal("state not changed") + } + }) + } + } +} diff --git a/pkg/controller/pxcrestore/helpers_test.go b/pkg/controller/pxcrestore/helpers_test.go index e1120ff5cd..8783f69750 100644 --- a/pkg/controller/pxcrestore/helpers_test.go +++ b/pkg/controller/pxcrestore/helpers_test.go @@ -149,15 +149,30 @@ func reconciler(cl client.Client) *ReconcilePerconaXtraDBClusterRestore { func buildFakeClient(objs ...runtime.Object) client.Client { s := scheme.Scheme - s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBClusterRestore)) - s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBClusterRestoreList)) - s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBClusterBackup)) - s.AddKnownTypes(api.SchemeGroupVersion, new(api.PerconaXtraDBCluster)) + types := []runtime.Object{ + new(api.PerconaXtraDBClusterRestore), + new(api.PerconaXtraDBClusterRestoreList), + new(api.PerconaXtraDBClusterBackup), + new(api.PerconaXtraDBCluster), + } + + s.AddKnownTypes(api.SchemeGroupVersion, types...) + + toClientObj := func(objs []runtime.Object) []client.Object { + cliObjs := make([]client.Object, 0, len(objs)) + for _, obj := range objs { + cliObj, ok := obj.(client.Object) + if ok { + cliObjs = append(cliObjs, cliObj) + } + } + return cliObjs + } cl := fake.NewClientBuilder(). WithScheme(s). WithRuntimeObjects(objs...). - WithStatusSubresource(&api.PerconaXtraDBClusterRestore{}). + WithStatusSubresource(toClientObj(types)...). Build() return cl diff --git a/pkg/controller/pxcrestore/restorer.go b/pkg/controller/pxcrestore/restorer.go index 9065379867..b3d89a6503 100644 --- a/pkg/controller/pxcrestore/restorer.go +++ b/pkg/controller/pxcrestore/restorer.go @@ -108,14 +108,8 @@ func (s *pvc) Validate(ctx context.Context) error { switch pod.Status.Phase { case corev1.PodFailed: - if err := s.k8sClient.Delete(ctx, pod); err != nil { - return errors.Wrap(err, "failed to delete") - } return errors.Errorf("backup files not found on %s", destination) case corev1.PodSucceeded: - if err := s.k8sClient.Delete(ctx, pod); err != nil { - return errors.Wrap(err, "failed to delete") - } return nil default: return errWaitValidate @@ -168,11 +162,11 @@ func (s *pvc) Init(ctx context.Context) error { } err = s.k8sClient.Create(ctx, svc) - if err != nil { + if client.IgnoreAlreadyExists(err) != nil { return errors.Wrap(err, "create service") } err = s.k8sClient.Create(ctx, pod) - if err != nil { + if client.IgnoreAlreadyExists(err) != nil { return errors.Wrap(err, "create pod") } } @@ -195,6 +189,10 @@ func (s *pvc) Finalize(ctx context.Context) error { if err := s.k8sClient.Delete(ctx, pod); client.IgnoreNotFound(err) != nil { return errors.Wrap(err, "failed to delete pvc pod") } + pod.Name += "-verify" + if err := s.k8sClient.Delete(ctx, pod); client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "failed to delete pvc pod") + } return nil } From 7b5a420742ce91caf7574135e02a02c633104621 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 7 Feb 2024 12:52:18 +0200 Subject: [PATCH 8/9] improvements --- pkg/controller/pxcrestore/controller.go | 5 ++--- pkg/controller/pxcrestore/restorer.go | 12 ++++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/controller/pxcrestore/controller.go b/pkg/controller/pxcrestore/controller.go index 63a4a3586e..114e32117a 100644 --- a/pkg/controller/pxcrestore/controller.go +++ b/pkg/controller/pxcrestore/controller.go @@ -194,8 +194,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re case api.RestoreStopCluster: err = stopCluster(ctx, r.client, cluster.DeepCopy()) if err != nil { - switch err { - case errWaitingPods, errWaitingPVC: + if errors.Is(err, errWaitingPods) || errors.Is(err, errWaitingPVC) { log.Info("waiting for cluster to stop", "cluster", cr.Spec.PXCCluster, "msg", err.Error()) return rr, nil } @@ -253,7 +252,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re } return rr, nil } else { - if cluster.Status.ObservedGeneration == cluster.Generation && cluster.Status.PXC.Status == api.AppStateReady { + if cluster.Status.ObservedGeneration != cluster.Generation || cluster.Status.PXC.Status != api.AppStateReady { log.Info("Waiting for cluster to start", "cluster", cluster.Name) return rr, nil } diff --git a/pkg/controller/pxcrestore/restorer.go b/pkg/controller/pxcrestore/restorer.go index b3d89a6503..7eb0697623 100644 --- a/pkg/controller/pxcrestore/restorer.go +++ b/pkg/controller/pxcrestore/restorer.go @@ -141,16 +141,12 @@ func (s *pvc) Init(ctx context.Context) error { initInProcess := true - if err := s.k8sClient.Get(ctx, types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, svc); err != nil { - if k8serrors.IsNotFound(err) { - initInProcess = false - } + if err := s.k8sClient.Get(ctx, types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, svc); k8serrors.IsNotFound(err) { + initInProcess = false } - if err := s.k8sClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: svc.Namespace}, pod); err != nil { - if k8serrors.IsNotFound(err) { - initInProcess = false - } + if err := s.k8sClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: svc.Namespace}, pod); k8serrors.IsNotFound(err) { + initInProcess = false } if !initInProcess { From 7ea4009384da3c0bcb6f6012505eeb488544bbd5 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 7 Feb 2024 14:34:50 +0200 Subject: [PATCH 9/9] add TODO comment --- pkg/controller/pxcrestore/controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/pxcrestore/controller.go b/pkg/controller/pxcrestore/controller.go index 114e32117a..9959efc718 100644 --- a/pkg/controller/pxcrestore/controller.go +++ b/pkg/controller/pxcrestore/controller.go @@ -90,6 +90,7 @@ func (r *ReconcilePerconaXtraDBClusterRestore) Reconcile(ctx context.Context, re log := logf.FromContext(ctx) rr := reconcile.Result{ + // TODO: do not depend on the RequeueAfter RequeueAfter: time.Second * 5, }