From 299f4ab8ad01137c463574fe013da30efc6b05d0 Mon Sep 17 00:00:00 2001 From: Shuo Wu Date: Wed, 9 Dec 2020 20:08:59 +0800 Subject: [PATCH] Longhorn v1.1.0 Longhorn #2061 Signed-off-by: Shuo Wu --- charts/longhorn/Chart.yaml | 7 +- charts/longhorn/README.md | 58 ++-- charts/longhorn/questions.yml | 304 ++++++++++++++---- charts/longhorn/templates/_helpers.tpl | 28 ++ charts/longhorn/templates/clusterrole.yaml | 15 +- .../templates/clusterrolebinding.yaml | 2 +- charts/longhorn/templates/crds.yaml | 295 +++++++++++++++-- charts/longhorn/templates/daemonset-sa.yaml | 34 +- .../longhorn/templates/default-setting.yaml | 16 +- .../longhorn/templates/deployment-driver.yaml | 66 ++-- charts/longhorn/templates/deployment-ui.yaml | 14 +- charts/longhorn/templates/ingress.yaml | 1 + .../longhorn/templates/postupgrade-job.yaml | 14 +- charts/longhorn/templates/psp.yaml | 10 +- charts/longhorn/templates/registry-secret.yml | 7 +- charts/longhorn/templates/serviceaccount.yaml | 2 +- charts/longhorn/templates/storageclass.yaml | 36 ++- charts/longhorn/templates/tls-secrets.yaml | 1 + charts/longhorn/templates/uninstall-job.yaml | 14 +- charts/longhorn/values.yaml | 73 ++++- 20 files changed, 735 insertions(+), 262 deletions(-) diff --git a/charts/longhorn/Chart.yaml b/charts/longhorn/Chart.yaml index ce617fec..8e15810e 100644 --- a/charts/longhorn/Chart.yaml +++ b/charts/longhorn/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: longhorn -version: 1.0.2 -appVersion: v1.0.2 -kubeVersion: ">=v1.14.0-r0" +version: 1.1.0 +appVersion: v1.1.0 +kubeVersion: ">=v1.16.0-r0" description: Longhorn is a distributed block storage system for Kubernetes. keywords: - longhorn @@ -16,6 +16,7 @@ sources: - https://github.com/longhorn/longhorn - https://github.com/longhorn/longhorn-engine - https://github.com/longhorn/longhorn-instance-manager +- https://github.com/longhorn/longhorn-share-manager - https://github.com/longhorn/longhorn-manager - https://github.com/longhorn/longhorn-ui - https://github.com/longhorn/longhorn-tests diff --git a/charts/longhorn/README.md b/charts/longhorn/README.md index 068bb5d6..d2f6d05e 100644 --- a/charts/longhorn/README.md +++ b/charts/longhorn/README.md @@ -1,48 +1,60 @@ -# Rancher Longhorn Chart +# Longhorn Chart -> **Important**: Please install Longhorn chart in `longhorn-system` namespace only. +> **Important**: Please install the Longhorn chart in the `longhorn-system` namespace only. > **Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. -The following document pertains to running Longhorn from the Rancher 2.0 chart. - ## Source Code Longhorn is 100% open source software. Project source code is spread across a number of repos: 1. Longhorn Engine -- Core controller/replica logic https://github.com/longhorn/longhorn-engine 2. Longhorn Instance Manager -- Controller/replica instance lifecycle management https://github.com/longhorn/longhorn-instance-manager -3. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager +3. Longhorn Share Manager -- NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes. https://github.com/longhorn/longhorn-share-manager +4. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager 4. Longhorn UI -- Dashboard https://github.com/longhorn/longhorn-ui ## Prerequisites -1. Rancher v2.1+ -2. Docker v1.13+ -3. Kubernetes v1.14+ -4. Make sure `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. -5. Make sure `open-iscsi` has been installed in all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. - -## Uninstallation - -1. To prevent damage to the Kubernetes cluster, we recommend deleting all Kubernetes workloads using Longhorn volumes (PersistentVolume, PersistentVolumeClaim, StorageClass, Deployment, StatefulSet, DaemonSet, etc). +1. Docker v1.13+ +2. Kubernetes v1.16+ +3. Make sure `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. +4. Make sure `open-iscsi` has been installed in all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. -2. From Rancher UI, navigate to `Catalog Apps` tab and delete Longhorn app. +## Installation +1. Add Longhorn chart repository. +``` +helm repo add longhorn https://charts.longhorn.io +``` -## Troubleshooting +2. Update local Longhorn chart information from chart repository. +``` +helm repo update +``` -### I deleted the Longhorn App from Rancher UI instead of following the uninstallation procedure +3. Install Longhorn chart. +- With Helm 2, the following command will create the `longhorn-system` namespace and install the Longhorn chart together. +``` +helm install longhorn/longhorn --name longhorn --namespace longhorn-system +``` +- With Helm 3, the following commands will create the `longhorn-system` namespace first, then install the Longhorn chart. -Redeploy the (same version) Longhorn App. Follow the uninstallation procedure above. +``` +kubectl create namespace longhorn-system +helm install longhorn longhorn/longhorn --namespace longhorn-system +``` -### Problems with CRDs +## Uninstallation -If your CRD instances or the CRDs themselves can't be deleted for whatever reason, run the commands below to clean up. Caution: this will wipe all Longhorn state! +With Helm 2 to uninstall Longhorn. +``` +helm delete longhorn --purge +``` +With Helm 3 to uninstall Longhorn. ``` -# Delete CRD instances and definitions -curl -s https://raw.githubusercontent.com/longhorn/longhorn/v0.8.1/scripts/cleanup.sh |bash -s v062 -curl -s https://raw.githubusercontent.com/longhorn/longhorn/v0.8.1/scripts/cleanup.sh |bash -s v070 +helm uninstall longhorn -n longhorn-system +kubectl delete namespace longhorn-system ``` --- diff --git a/charts/longhorn/questions.yml b/charts/longhorn/questions.yml index 61a5033a..ef33761f 100644 --- a/charts/longhorn/questions.yml +++ b/charts/longhorn/questions.yml @@ -10,105 +10,129 @@ questions: show_subquestion_if: false group: "Longhorn Images" subquestions: - - variable: image.longhorn.manager + - variable: image.longhorn.manager.repository default: longhornio/longhorn-manager - description: "Specify Longhorn Manager Image Name" + description: "Specify Longhorn Manager Image Repository" type: string - label: Longhorn Manager Image Name + label: Longhorn Manager Image Repository group: "Longhorn Images Settings" - - variable: image.longhorn.managerTag - default: v1.0.2 + - variable: image.longhorn.manager.tag + default: v1.1.0 description: "Specify Longhorn Manager Image Tag" type: string label: Longhorn Manager Image Tag group: "Longhorn Images Settings" - - variable: image.longhorn.engine + - variable: image.longhorn.engine.repository default: longhornio/longhorn-engine - description: "Specify Longhorn Engine Image Name" + description: "Specify Longhorn Engine Image Repository" type: string - label: Longhorn Engine Image Name + label: Longhorn Engine Image Repository group: "Longhorn Images Settings" - - variable: image.longhorn.engineTag - default: v1.0.2 + - variable: image.longhorn.engine.tag + default: v1.1.0 description: "Specify Longhorn Engine Image Tag" type: string label: Longhorn Engine Image Tag group: "Longhorn Images Settings" - - variable: image.longhorn.ui + - variable: image.longhorn.ui.repository default: longhornio/longhorn-ui - description: "Specify Longhorn UI Image Name" + description: "Specify Longhorn UI Image Repository" type: string - label: Longhorn UI Image Name + label: Longhorn UI Image Repository group: "Longhorn Images Settings" - - variable: image.longhorn.uiTag - default: v1.0.2 + - variable: image.longhorn.ui.tag + default: v1.1.0 description: "Specify Longhorn UI Image Tag" type: string label: Longhorn UI Image Tag group: "Longhorn Images Settings" - - variable: image.longhorn.instanceManager + - variable: image.longhorn.instanceManager.repository default: longhornio/longhorn-instance-manager - description: "Specify Longhorn Instance Manager Image Name" + description: "Specify Longhorn Instance Manager Image Repository" type: string - label: Longhorn Instance Manager Image Name + label: Longhorn Instance Manager Image Repository group: "Longhorn Images Settings" - - variable: image.longhorn.instanceManagerTag - default: v1_20200514 + - variable: image.longhorn.instanceManager.tag + default: v1_20201216 description: "Specify Longhorn Instance Manager Image Tag" type: string label: Longhorn Instance Manager Image Tag group: "Longhorn Images Settings" - - variable: csi.attacherImage + - variable: image.longhorn.shareManager.repository + default: longhornio/longhorn-share-manager + description: "Specify Longhorn Share Manager Image Repository" + type: string + label: Longhorn Share Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.shareManager.tag + default: v1_20201204 + description: "Specify Longhorn Share Manager Image Tag" + type: string + label: Longhorn Share Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.csi.attacher.repository default: longhornio/csi-attacher - description: "Specify CSI attacher image. Leave blank to autodetect." + description: "Specify CSI attacher image repository. Leave blank to autodetect." type: string - label: Longhorn CSI Attacher Image + label: Longhorn CSI Attacher Image Repository group: "Longhorn CSI Driver Images" - - variable: csi.attacherImageTag - default: v2.0.0 + - variable: image.csi.attacher.tag + default: v2.2.1-lh1 description: "Specify CSI attacher image tag. Leave blank to autodetect." type: string label: Longhorn CSI Attacher Image Tag group: "Longhorn CSI Driver Images" - - variable: csi.provisionerImage + - variable: image.csi.provisioner.repository default: longhornio/csi-provisioner - description: "Specify CSI provisioner image. Leave blank to autodetect." + description: "Specify CSI provisioner image repository. Leave blank to autodetect." type: string - label: Longhorn CSI Provisioner Image + label: Longhorn CSI Provisioner Image Repository group: "Longhorn CSI Driver Images" - - variable: csi.provisionerImageTag - default: v1.4.0 + - variable: image.csi.provisioner.tag + default: v1.6.0-lh1 description: "Specify CSI provisioner image tag. Leave blank to autodetect." type: string label: Longhorn CSI Provisioner Image Tag group: "Longhorn CSI Driver Images" - - variable: csi.nodeDriverRegistrarImage + - variable: image.csi.nodeDriverRegistrar.repository default: longhornio/csi-node-driver-registrar - description: "Specify CSI Node Driver Registrar image. Leave blank to autodetect." + description: "Specify CSI Node Driver Registrar image repository. Leave blank to autodetect." type: string - label: Longhorn CSI Node Driver Registrar Image + label: Longhorn CSI Node Driver Registrar Image Repository group: "Longhorn CSI Driver Images" - - variable: csi.nodeDriverRegistrarImageTag - default: v1.2.0 + - variable: image.csi.nodeDriverRegistrar.tag + default: v1.2.0-lh1 description: "Specify CSI Node Driver Registrar image tag. Leave blank to autodetect." type: string label: Longhorn CSI Node Driver Registrar Image Tag group: "Longhorn CSI Driver Images" - - variable: csi.resizerImage + - variable: image.csi.resizer.repository default: longhornio/csi-resizer - description: "Specify CSI Driver Resizer image. Leave blank to autodetect." + description: "Specify CSI Driver Resizer image repository. Leave blank to autodetect." type: string - label: Longhorn CSI Driver Resizer Image + label: Longhorn CSI Driver Resizer Image Repository group: "Longhorn CSI Driver Images" - - variable: csi.resizerImageTag - default: v0.3.0 + - variable: image.csi.resizer.tag + default: v0.5.1-lh1 description: "Specify CSI Driver Resizer image tag. Leave blank to autodetect." type: string label: Longhorn CSI Driver Resizer Image Tag group: "Longhorn CSI Driver Images" + - variable: image.csi.snapshotter.repository + default: longhornio/csi-snapshotter + description: "Specify CSI Driver Snapshotter image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Driver Snapshotter Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.snapshotter.tag + default: v2.1.1-lh1 + description: "Specify CSI Driver Snapshotter image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Driver Snapshotter Image Tag + group: "Longhorn CSI Driver Images" - variable: privateRegistry.registryUrl label: Private registry URL - description: "URL of private registry" + description: "URL of private registry. Leave blank to apply system default registry." group: "Private Registry Settings" type: string default: "" @@ -124,6 +148,12 @@ questions: group: "Private Registry Settings" type: password default: "" +- variable: privateRegistry.registrySecret + label: Private registry secret name + description: "Longhorn will automatically generate a Kubernetes secret with this name and use it to pull images from your private registry." + group: "Private Registry Settings" + type: string + default: "" - variable: longhorn.default_setting default: "false" description: "Customize the default settings before installing Longhorn for the first time. This option will only work if the cluster hasn't installed Longhorn." @@ -132,12 +162,6 @@ questions: show_subquestion_if: true group: "Longhorn Default Settings" subquestions: - - variable: defaultSettings.registrySecret - label: Private registry secret - description: "The Kubernetes Secret name" - group: "Longhorn Default Settings" - type: string - default: "" - variable: csi.kubeletRootDir default: description: "Specify kubelet root-dir. Leave blank to autodetect." @@ -146,7 +170,7 @@ questions: group: "Longhorn CSI Driver Settings" - variable: csi.attacherReplicaCount type: int - default: + default: 3 min: 1 max: 10 description: "Specify replica count of CSI Attacher. By default 3." @@ -154,27 +178,28 @@ questions: group: "Longhorn CSI Driver Settings" - variable: csi.provisionerReplicaCount type: int - default: + default: 3 min: 1 max: 10 description: "Specify replica count of CSI Provisioner. By default 3." label: Longhorn CSI Provisioner replica count group: "Longhorn CSI Driver Settings" - - variable: persistence.defaultClass - default: "true" - description: "Set as default StorageClass" - group: "Longhorn CSI Driver Settings" - type: boolean - required: true - label: Default Storage Class - - variable: persistence.defaultClassReplicaCount - description: "Set replica count for default StorageClass" + - variable: csi.resizerReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Specify replica count of CSI Resizer. By default 3." + label: Longhorn CSI Resizer replica count group: "Longhorn CSI Driver Settings" + - variable: csi.snapshotterReplicaCount type: int default: 3 min: 1 max: 10 - label: Default Storage Class Replica Count + description: "Specify replica count of CSI Snapshotter. By default 3." + label: Longhorn CSI Snapshotter replica count + group: "Longhorn CSI Driver Settings" - variable: defaultSettings.backupTarget label: Backup Target description: "The endpoint used to access the backupstore. NFS and S3 are supported." @@ -187,6 +212,13 @@ questions: group: "Longhorn Default Settings" type: string default: + - variable: defaultSettings.allowRecurringJobWhileVolumeDetached + label: Allow Recurring Job While Volume Is Detached + description: 'If this setting is enabled, Longhorn will automatically attaches the volume and takes snapshot/backup when it is the time to do recurring snapshot/backup. +Note that the volume is not ready for workload during the period when the volume was automatically attached. Workload will have to wait until the recurring job finishes.' + group: "Longhorn Default Settings" + type: boolean + default: "false" - variable: defaultSettings.createDefaultDiskLabeledNodes label: Create Default Disk on Labeled Nodes description: 'Create default Disk automatically only on Nodes with the label "node.longhorn.io/create-default-disk=true" if no other disks exist. If disabled, the default disk will be created on all new nodes when each node is first added.' @@ -199,6 +231,19 @@ questions: group: "Longhorn Default Settings" type: string default: "/var/lib/longhorn/" + - variable: defaultSettings.defaultDataLocality + label: Default Data Locality + description: 'We say a Longhorn volume has data locality if there is a local replica of the volume on the same node as the pod which is using the volume. +This setting specifies the default data locality when a volume is created from the Longhorn UI. For Kubernetes configuration, update the `dataLocality` in the StorageClass +The available modes are: +- **disabled**. This is the default option. There may or may not be a replica on the same node as the attached volume (workload) +- **best-effort**. This option instructs Longhorn to try to keep a replica on the same node as the attached volume (workload). Longhorn will not stop the volume, even if it cannot keep a replica local to the attached volume (workload) due to environment limitation, e.g. not enough disk space, incompatible disk tags, etc.' + group: "Longhorn Default Settings" + type: enum + options: + - "disabled" + - "best-effort" + default: "disabled" - variable: defaultSettings.replicaSoftAntiAffinity label: Replica Node Level Soft Anti-Affinity description: 'Allow scheduling on nodes with existing healthy replicas of the same volume. By default false.' @@ -236,10 +281,13 @@ questions: default: 3 - variable: defaultSettings.guaranteedEngineCPU label: Guaranteed Engine CPU - description: 'Allow Longhorn Instance Managers to have guaranteed CPU allocation. The value is how many CPUs should be reserved for each Engine/Replica Instance Manager Pod created by Longhorn. For example, 0.1 means one-tenth of a CPU. This will help maintain engine stability during high node workload. It only applies to the Engine/Replica Manager Pods created after the setting took effect. -WARNING: After this setting is changed, all the instance managers on all the nodes will be automatically restarted. -WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES. -By default 0.25.' + description: "Allow Longhorn Instance Managers to have guaranteed CPU allocation. By default 0.25. The value is how many CPUs should be reserved for each Engine/Replica Instance Manager Pod created by Longhorn. For example, 0.1 means one-tenth of a CPU. This will help maintain engine stability during high node workload. It only applies to the Engine/Replica Instance Manager Pods created after the setting took effect. +In order to prevent unexpected volume crash, you can use the following formula to calculate an appropriate value for this setting: +'Guaranteed Engine CPU = The estimated max Longhorn volume/replica count on a node * 0.1'. +The result of above calculation doesn't mean that's the maximum CPU resources the Longhorn workloads require. To fully exploit the Longhorn volume I/O performance, you can allocate/guarantee more CPU resources via this setting. +If it's hard to estimate the volume/replica count now, you can leave it with the default value, or allocate 1/8 of total CPU of a node. Then you can tune it when there is no running workload using Longhorn volumes. +WARNING: After this setting is changed, all the instance managers on all the nodes will be automatically restarted +WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES." group: "Longhorn Default Settings" type: float default: 0.25 @@ -258,8 +306,15 @@ By default 0.25.' default: 300 - variable: defaultSettings.taintToleration label: Kubernetes Taint Toleration - description: 'To dedicate nodes to store Longhorn replicas and reject other general workloads, set tolerations for Longhorn and add taints for the storage nodes. All Longhorn volumes should be detached before modifying toleration settings. We recommend setting tolerations during Longhorn deployment because the Longhorn system cannot be operated during the update. Multiple tolerations can be set here, and these tolerations are separated by semicolon. For example, `key1=value1:NoSchedule; key2:NoExecute`. Because `kubernetes.io` is used as the key of all Kubernetes default tolerations, it should not be used in the toleration settings. -WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.' + description: "To dedicate nodes to store Longhorn replicas and reject other general workloads, set tolerations for Longhorn and add taints for the storage nodes. +All Longhorn volumes should be detached before modifying toleration settings. +We recommend setting tolerations during Longhorn deployment because the Longhorn system cannot be operated during the update. +Multiple tolerations can be set here, and these tolerations are separated by semicolon. For example: +* `key1=value1:NoSchedule; key2:NoExecute` +* `:` this toleration tolerates everything because an empty key with operator `Exists` matches all keys, values and effects +* `key1=value1:` this toleration has empty effect. It matches all effects with key `key1` +Because `kubernetes.io` is used as the key of all Kubernetes default tolerations, it should not be used in the toleration settings. +WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES!" group: "Longhorn Default Settings" type: string default: "" @@ -275,6 +330,14 @@ WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.' group: "Longhorn Default Settings" type: boolean default: "true" + - variable: defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly + label: Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly + description: 'If enabled, Longhorn will automatically delete the workload pod that is managed by a controller (e.g. deployment, statefulset, daemonset, etc...) when Longhorn volume is detached unexpectedly (e.g. during Kubernetes upgrade, Docker reboot, or network disconnect). By deleting the pod, its controller restarts the pod and Kubernetes handles volume reattachment and remount. +If disabled, Longhorn will not delete the workload pod that is managed by a controller. You will have to manually restart the pod to reattach and remount the volume. +**Note:** This setting does not apply to the workload pods that do not have a controller. Longhorn never deletes them.' + group: "Longhorn Default Settings" + type: boolean + default: "true" - variable: defaultSettings.disableSchedulingOnCordonedNode label: Disable Scheduling On Cordoned Node description: "Disable Longhorn manager to schedule replica on Kubernetes cordoned node. By default true." @@ -293,15 +356,118 @@ WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.' group: "Longhorn Default Settings" type: enum options: - - "wait" - - "never" - - "immediate" + - "wait" + - "never" + - "immediate" default: "wait" + - variable: defaultSettings.nodeDownPodDeletionPolicy + label: Pod Deletion Policy When Node is Down + description: "Defines the Longhorn action when a Volume is stuck with a StatefulSet/Deployment Pod on a node that is down. +- **do-nothing** is the default Kubernetes behavior of never force deleting StatefulSet/Deployment terminating pods. Since the pod on the node that is down isn't removed, Longhorn volumes are stuck on nodes that are down. +- **delete-statefulset-pod** Longhorn will force delete StatefulSet terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods. +- **delete-deployment-pod** Longhorn will force delete Deployment terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods. +- **delete-both-statefulset-and-deployment-pod** Longhorn will force delete StatefulSet/Deployment terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods." + group: "Longhorn Default Settings" + type: enum + options: + - "do-nothing" + - "delete-statefulset-pod" + - "delete-deployment-pod" + - "delete-both-statefulset-and-deployment-pod" + default: "do-nothing" + - variable: defaultSettings.allowNodeDrainWithLastHealthyReplica + label: Allow Node Drain with the Last Healthy Replica + description: "By default, Longhorn will block `kubectl drain` action on a node if the node contains the last healthy replica of a volume. +If this setting is enabled, Longhorn will **not** block `kubectl drain` action on a node even if the node contains the last healthy replica of a volume." + group: "Longhorn Default Settings" + type: boolean + default: "false" - variable: defaultSettings.mkfsExt4Parameters label: Custom mkfs.ext4 parameters description: "Allows setting additional filesystem creation parameters for ext4. For older host kernels it might be necessary to disable the optional ext4 metadata_csum feature by specifying `-O ^64bit,^metadata_csum`." group: "Longhorn Default Settings" type: string + - variable: defaultSettings.disableReplicaRebuild + label: Disable Replica Rebuild + description: "This setting disable replica rebuild cross the whole cluster, eviction and data locality feature won't work if this setting is true. But doesn't have any impact to any current replica rebuild and restore disaster recovery volume." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.replicaReplenishmentWaitInterval + label: Replica Replenishment Wait Interval + description: "In seconds. The interval determines how long Longhorn will wait at least in order to reuse the existing data on a failed replica rather than directly creating a new replica for a degraded volume. +Warning: This option works only when there is a failed replica in the volume. And this option may block the rebuilding for a while in the case." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 600 + - variable: defaultSettings.disableRevisionCounter + label: Disable Revision Counter + description: "This setting is only for volumes created by UI. By default, this is false meaning there will be a reivision counter file to track every write to the volume. During salvage recovering Longhorn will pick the repica with largest reivision counter as candidate to recover the whole volume. If revision counter is disabled, Longhorn will not track every write to the volume. During the salvage recovering, Longhorn will use the 'volume-head-xxx.img' file last modification time and file size to pick the replica candidate to recover the whole volume." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.systemManagedPodsImagePullPolicy + label: System Managed Pod Image Pull Policy + description: "This setting defines the Image Pull Policy of Longhorn system managed pods, e.g. instance manager, engine image, CSI driver, etc. The new Image Pull Policy will only apply after the system managed pods restart." + group: "Longhorn Default Settings" + type: enum + options: + - "if-not-present" + - "always" + - "never" + default: "if-not-present" + - variable: defaultSettings.allowVolumeCreationWithDegradedAvailability + label: Allow Volume Creation with Degraded Availability + description: "This setting allows user to create and attach a volume that doesn't have all the replicas scheduled at the time of creation." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.autoCleanupSystemGeneratedSnapshot + label: Automatically Cleanup System Generated Snapshot + description: "This setting enables Longhorn to automatically cleanup the system generated snapshot after replica rebuild is done." + group: "Longhorn Default Settings" + type: boolean + default: "true" +- variable: persistence.defaultClass + default: "true" + description: "Set as default StorageClass for Longhorn" + label: Default Storage Class + group: "Longhorn Storage Class Settings" + required: true + type: boolean +- variable: persistence.reclaimPolicy + label: Storage Class Retain Policy + description: "Define reclaim policy (Retain or Delete)" + group: "Longhorn Storage Class Settings" + required: true + type: enum + options: + - "Delete" + - "Retain" + default: "Delete" +- variable: persistence.defaultClassReplicaCount + description: "Set replica count for Longhorn StorageClass" + label: Default Storage Class Replica Count + group: "Longhorn Storage Class Settings" + type: int + min: 1 + max: 10 + default: 3 +- variable: persistence.recurringJobs.enable + description: "Enable recurring job for Longhorn StorageClass" + group: "Longhorn Storage Class Settings" + label: Enable Storage Class Recurring Job + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.recurringJobs.jobList + description: 'Recurring job list for Longhorn StorageClass. Please be careful of quotes of input. e.g., [{"name":"backup", "task":"backup", "cron":"*/2 * * * *", "retain":1,"labels": {"interval":"2m"}}]' + label: Storage Class Recurring Job List + group: "Longhorn Storage Class Settings" + type: string + default: - variable: ingress.enabled default: "false" description: "Expose app using Layer 7 Load Balancer - ingress" @@ -343,4 +509,4 @@ WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.' description: "Setup a pod security policy for Longhorn workloads." label: Pod Security Policy type: boolean - group: "Other Settings" \ No newline at end of file + group: "Other Settings" diff --git a/charts/longhorn/templates/_helpers.tpl b/charts/longhorn/templates/_helpers.tpl index 7ed4536e..3fbc2ac0 100644 --- a/charts/longhorn/templates/_helpers.tpl +++ b/charts/longhorn/templates/_helpers.tpl @@ -36,3 +36,31 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} {{- end -}} + + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "registry_url" -}} +{{- if .Values.privateRegistry.registryUrl -}} +{{- printf "%s/" .Values.privateRegistry.registryUrl -}} +{{- else -}} +{{ include "system_default_registry" . }} +{{- end -}} +{{- end -}} + +{{- /* + define the longhorn release namespace +*/ -}} +{{- define "release_namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} diff --git a/charts/longhorn/templates/clusterrole.yaml b/charts/longhorn/templates/clusterrole.yaml index 491c3e6a..c6976175 100644 --- a/charts/longhorn/templates/clusterrole.yaml +++ b/charts/longhorn/templates/clusterrole.yaml @@ -22,19 +22,26 @@ rules: - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["*"] - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["watch", "list"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "volumeattachments", "csinodes", "csidrivers"] verbs: ["*"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] - apiGroups: ["longhorn.io"] resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", - "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status"] + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status"] verbs: ["*"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] diff --git a/charts/longhorn/templates/clusterrolebinding.yaml b/charts/longhorn/templates/clusterrolebinding.yaml index 30c7fa78..66ac62f9 100644 --- a/charts/longhorn/templates/clusterrolebinding.yaml +++ b/charts/longhorn/templates/clusterrolebinding.yaml @@ -10,4 +10,4 @@ roleRef: subjects: - kind: ServiceAccount name: longhorn-service-account - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} diff --git a/charts/longhorn/templates/crds.yaml b/charts/longhorn/templates/crds.yaml index 8e4be3d3..04cf9ed0 100644 --- a/charts/longhorn/templates/crds.yaml +++ b/charts/longhorn/templates/crds.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -14,11 +14,42 @@ spec: - lhe singular: engine scope: Namespaced - version: v1beta1 - subresources: - status: {} + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: State + type: string + description: The current state of the engine + jsonPath: .status.currentState + - name: Node + type: string + description: The node that the engine is on + jsonPath: .spec.nodeID + - name: InstanceManager + type: string + description: The instance manager of the engine + jsonPath: .status.instanceManagerName + - name: Image + type: string + description: The current image of the engine + jsonPath: .status.currentImage + - name: Age + type: date + jsonPath: .metadata.creationTimestamp --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -34,11 +65,46 @@ spec: - lhr singular: replica scope: Namespaced - version: v1beta1 - subresources: - status: {} + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: State + type: string + description: The current state of the replica + jsonPath: .status.currentState + - name: Node + type: string + description: The node that the replica is on + jsonPath: .spec.nodeID + - name: Disk + type: string + description: The disk that the replica is on + jsonPath: .spec.diskID + - name: InstanceManager + type: string + description: The instance manager of the replica + jsonPath: .status.instanceManagerName + - name: Image + type: string + description: The current image of the replica + jsonPath: .status.currentImage + - name: Age + type: date + jsonPath: .metadata.creationTimestamp --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -54,9 +120,23 @@ spec: - lhs singular: setting scope: Namespaced - version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: Value + type: string + description: The value of the setting + jsonPath: .value + - name: Age + type: date + jsonPath: .metadata.creationTimestamp --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -72,11 +152,46 @@ spec: - lhv singular: volume scope: Namespaced - version: v1beta1 - subresources: - status: {} + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: State + type: string + description: The state of the volume + jsonPath: .status.state + - name: Robustness + type: string + description: The robustness of the volume + jsonPath: .status.robustness + - name: Scheduled + type: string + description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + - name: Size + type: string + description: The size of the volume + jsonPath: .spec.size + - name: Node + type: string + description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + - name: Age + type: date + jsonPath: .metadata.creationTimestamp --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -92,11 +207,42 @@ spec: - lhei singular: engineimage scope: Namespaced - version: v1beta1 - subresources: - status: {} + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: State + type: string + description: State of the engine image + jsonPath: .status.state + - name: Image + type: string + description: The Longhorn engine image + jsonPath: .spec.image + - name: RefCount + type: integer + description: Number of volumes are using the engine image + jsonPath: .status.refCount + - name: BuildDate + type: date + description: The build date of the engine image + jsonPath: .status.buildDate + - name: Age + type: date + jsonPath: .metadata.creationTimestamp --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -112,11 +258,38 @@ spec: - lhn singular: node scope: Namespaced - version: v1beta1 - subresources: - status: {} + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: Ready + type: string + description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + - name: AllowScheduling + type: boolean + description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + - name: Schedulable + type: string + description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + - name: Age + type: date + jsonPath: .metadata.creationTimestamp --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} @@ -132,6 +305,76 @@ spec: - lhim singular: instancemanager scope: Namespaced - version: v1beta1 - subresources: - status: {} + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: State + type: string + description: The state of the instance manager + jsonPath: .status.currentState + - name: Type + type: string + description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + - name: Node + type: string + description: The node that the instance manager is running on + jsonPath: .spec.nodeID + - name: Age + type: date + jsonPath: .metadata.creationTimestamp +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: ShareManager + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: State + type: string + description: The state of the share manager + jsonPath: .status.state + - name: Node + type: string + description: The node that the share manager is owned by + jsonPath: .status.ownerID + - name: Age + type: date + jsonPath: .metadata.creationTimestamp diff --git a/charts/longhorn/templates/daemonset-sa.yaml b/charts/longhorn/templates/daemonset-sa.yaml index 5282dd37..e4079359 100644 --- a/charts/longhorn/templates/daemonset-sa.yaml +++ b/charts/longhorn/templates/daemonset-sa.yaml @@ -4,7 +4,7 @@ metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-manager name: longhorn-manager - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} spec: selector: matchLabels: @@ -16,11 +16,7 @@ spec: spec: containers: - name: longhorn-manager - {{- if .Values.privateRegistry.registryUrl }} - image: "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- else }} - image: "{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- end }} + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: IfNotPresent securityContext: privileged: true @@ -29,23 +25,13 @@ spec: - -d - daemon - --engine-image - {{- if .Values.privateRegistry.registryUrl }} - - "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.engine }}:{{ .Values.image.longhorn.engineTag }}" - {{- else }} - - "{{ .Values.image.longhorn.engine }}:{{ .Values.image.longhorn.engineTag }}" - {{- end }} + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.engine.repository }}:{{ .Values.image.longhorn.engine.tag }}" - --instance-manager-image - {{- if .Values.privateRegistry.registryUrl }} - - "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.instanceManager }}:{{ .Values.image.longhorn.instanceManagerTag }}" - {{- else }} - - "{{ .Values.image.longhorn.instanceManager }}:{{ .Values.image.longhorn.instanceManagerTag }}" - {{- end }} + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.instanceManager.repository }}:{{ .Values.image.longhorn.instanceManager.tag }}" + - --share-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.shareManager.repository }}:{{ .Values.image.longhorn.shareManager.tag }}" - --manager-image - {{- if .Values.privateRegistry.registryUrl }} - - "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- else }} - - "{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- end }} + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" - --service-account - longhorn-service-account ports: @@ -98,9 +84,9 @@ spec: - name: longhorn-default-setting configMap: name: longhorn-default-setting - {{- if .Values.defaultSettings.registrySecret }} + {{- if .Values.privateRegistry.registrySecret }} imagePullSecrets: - - name: {{ .Values.defaultSettings.registrySecret }} + - name: {{ .Values.privateRegistry.registrySecret }} {{- end }} serviceAccountName: longhorn-service-account updateStrategy: @@ -113,7 +99,7 @@ metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-manager name: longhorn-backend - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} spec: type: {{ .Values.service.manager.type }} sessionAffinity: ClientIP diff --git a/charts/longhorn/templates/default-setting.yaml b/charts/longhorn/templates/default-setting.yaml index 2eb10b4d..14c264e2 100644 --- a/charts/longhorn/templates/default-setting.yaml +++ b/charts/longhorn/templates/default-setting.yaml @@ -2,12 +2,13 @@ apiVersion: v1 kind: ConfigMap metadata: name: longhorn-default-setting - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} data: default-setting.yaml: |- backup-target: {{ .Values.defaultSettings.backupTarget }} backup-target-credential-secret: {{ .Values.defaultSettings.backupTargetCredentialSecret }} + allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }} create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }} default-data-path: {{ .Values.defaultSettings.defaultDataPath }} replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }} @@ -15,14 +16,23 @@ data: storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage }} upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }} default-replica-count: {{ .Values.defaultSettings.defaultReplicaCount }} + default-data-locality: {{ .Values.defaultSettings.defaultDataLocality }} guaranteed-engine-cpu: {{ .Values.defaultSettings.guaranteedEngineCPU }} default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass }} backupstore-poll-interval: {{ .Values.defaultSettings.backupstorePollInterval }} taint-toleration: {{ .Values.defaultSettings.taintToleration }} priority-class: {{ .Values.defaultSettings.priorityClass }} - registry-secret: {{ .Values.defaultSettings.registrySecret }} auto-salvage: {{ .Values.defaultSettings.autoSalvage }} + auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }} disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }} replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }} volume-attachment-recovery-policy: {{ .Values.defaultSettings.volumeAttachmentRecoveryPolicy }} - mkfs-ext4-parameters: {{ .Values.defaultSettings.mkfsExt4Parameters }} + node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy }} + allow-node-drain-with-last-healthy-replica: {{ .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica }} + mkfs-ext4-parameters: {{ .Values.defaultSettings.mkfsExt4Parameters }} + disable-replica-rebuild: {{ .Values.defaultSettings.disableReplicaRebuild }} + replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval }} + disable-revision-counter: {{ .Values.defaultSettings.disableRevisionCounter }} + system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy }} + allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }} + auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }} diff --git a/charts/longhorn/templates/deployment-driver.yaml b/charts/longhorn/templates/deployment-driver.yaml index 43c1af09..c4b6e358 100644 --- a/charts/longhorn/templates/deployment-driver.yaml +++ b/charts/longhorn/templates/deployment-driver.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: longhorn-driver-deployer - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: replicas: 1 @@ -16,30 +16,18 @@ spec: spec: initContainers: - name: wait-longhorn-manager - {{- if .Values.privateRegistry.registryUrl }} - image: {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.image.longhorn.manager .Values.image.longhorn.managerTag | quote }} - {{- else }} - image: {{ printf "%s:%s" .Values.image.longhorn.manager .Values.image.longhorn.managerTag | quote }} - {{- end }} + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] containers: - name: longhorn-driver-deployer - {{- if .Values.privateRegistry.registryUrl }} - image: {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.image.longhorn.manager .Values.image.longhorn.managerTag | quote }} - {{- else }} - image: {{ printf "%s:%s" .Values.image.longhorn.manager .Values.image.longhorn.managerTag | quote }} - {{- end }} + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} imagePullPolicy: IfNotPresent command: - longhorn-manager - -d - deploy-driver - --manager-image - {{- if .Values.privateRegistry.registryUrl }} - - {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.image.longhorn.manager .Values.image.longhorn.managerTag | quote }} - {{- else }} - - {{ printf "%s:%s" .Values.image.longhorn.manager .Values.image.longhorn.managerTag | quote }} - {{- end }} + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" - --manager-url - http://longhorn-backend:9500/v1 env: @@ -59,41 +47,26 @@ spec: - name: KUBELET_ROOT_DIR value: {{ .Values.csi.kubeletRootDir }} {{- end }} - {{- if .Values.privateRegistry.registryUrl }} - {{- if and .Values.csi.attacherImage .Values.csi.attacherImageTag }} + {{- if and .Values.image.csi.attacher.repository .Values.image.csi.attacher.tag }} - name: CSI_ATTACHER_IMAGE - value: {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.csi.attacherImage .Values.csi.attacherImageTag | quote }} + value: "{{ template "registry_url" . }}{{ .Values.image.csi.attacher.repository }}:{{ .Values.image.csi.attacher.tag }}" {{- end }} - {{- if and .Values.csi.provisionerImage .Values.csi.provisionerImageTag }} + {{- if and .Values.image.csi.provisioner.repository .Values.image.csi.provisioner.tag }} - name: CSI_PROVISIONER_IMAGE - value: {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.csi.provisionerImage .Values.csi.provisionerImageTag | quote }} + value: "{{ template "registry_url" . }}{{ .Values.image.csi.provisioner.repository }}:{{ .Values.image.csi.provisioner.tag }}" {{- end }} - {{- if and .Values.csi.nodeDriverRegistrarImage .Values.csi.nodeDriverRegistrarImageTag }} + {{- if and .Values.image.csi.nodeDriverRegistrar.repository .Values.image.csi.nodeDriverRegistrar.tag }} - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE - value: {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.csi.nodeDriverRegistrarImage .Values.csi.nodeDriverRegistrarImageTag | quote }} + value: "{{ template "registry_url" . }}{{ .Values.image.csi.nodeDriverRegistrar.repository }}:{{ .Values.image.csi.nodeDriverRegistrar.tag }}" {{- end }} - {{- if and .Values.csi.resizerImage .Values.csi.resizerImageTag }} + {{- if and .Values.image.csi.resizer.repository .Values.image.csi.resizer.tag }} - name: CSI_RESIZER_IMAGE - value: {{ printf "%s/%s:%s" .Values.privateRegistry.registryUrl .Values.csi.resizerImage .Values.csi.resizerImageTag | quote }} - {{- end }} - {{- else }} - {{- if and .Values.csi.attacherImage .Values.csi.attacherImageTag }} - - name: CSI_ATTACHER_IMAGE - value: {{ printf "%s:%s" .Values.csi.attacherImage .Values.csi.attacherImageTag | quote }} - {{- end }} - {{- if and .Values.csi.provisionerImage .Values.csi.provisionerImageTag }} - - name: CSI_PROVISIONER_IMAGE - value: {{ printf "%s:%s" .Values.csi.provisionerImage .Values.csi.provisionerImageTag | quote }} - {{- end }} - {{- if and .Values.csi.nodeDriverRegistrarImage .Values.csi.nodeDriverRegistrarImageTag }} - - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE - value: {{ printf "%s:%s" .Values.csi.nodeDriverRegistrarImage .Values.csi.nodeDriverRegistrarImageTag | quote }} + value: "{{ template "registry_url" . }}{{ .Values.image.csi.resizer.repository }}:{{ .Values.image.csi.resizer.tag }}" {{- end }} - {{- if and .Values.csi.resizerImage .Values.csi.resizerImageTag }} - - name: CSI_RESIZER_IMAGE - value: {{ printf "%s:%s" .Values.csi.resizerImage .Values.csi.resizerImageTag | quote }} + {{- if and .Values.image.csi.snapshotter.repository .Values.image.csi.snapshotter.tag }} + - name: CSI_SNAPSHOTTER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.snapshotter.repository }}:{{ .Values.image.csi.snapshotter.tag }}" {{- end }} - {{- end }} {{- if .Values.csi.attacherReplicaCount }} - name: CSI_ATTACHER_REPLICA_COUNT value: {{ .Values.csi.attacherReplicaCount | quote }} @@ -106,9 +79,14 @@ spec: - name: CSI_RESIZER_REPLICA_COUNT value: {{ .Values.csi.resizerReplicaCount | quote }} {{- end }} - {{- if .Values.defaultSettings.registrySecret }} + {{- if .Values.csi.snapshotterReplicaCount }} + - name: CSI_SNAPSHOTTER_REPLICA_COUNT + value: {{ .Values.csi.snapshotterReplicaCount | quote }} + {{- end }} + + {{- if .Values.privateRegistry.registrySecret }} imagePullSecrets: - - name: {{ .Values.defaultSettings.registrySecret }} + - name: {{ .Values.privateRegistry.registrySecret }} {{- end }} serviceAccountName: longhorn-service-account securityContext: diff --git a/charts/longhorn/templates/deployment-ui.yaml b/charts/longhorn/templates/deployment-ui.yaml index f75671be..da7c0ea5 100644 --- a/charts/longhorn/templates/deployment-ui.yaml +++ b/charts/longhorn/templates/deployment-ui.yaml @@ -4,7 +4,7 @@ metadata: labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ui name: longhorn-ui - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} spec: replicas: 1 selector: @@ -17,11 +17,7 @@ spec: spec: containers: - name: longhorn-ui - {{- if .Values.privateRegistry.registryUrl }} - image: "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.ui }}:{{ .Values.image.longhorn.uiTag }}" - {{- else }} - image: "{{ .Values.image.longhorn.ui }}:{{ .Values.image.longhorn.uiTag }}" - {{- end }} + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.ui.repository }}:{{ .Values.image.longhorn.ui.tag }} imagePullPolicy: IfNotPresent securityContext: runAsUser: 0 @@ -31,9 +27,9 @@ spec: env: - name: LONGHORN_MANAGER_IP value: "http://longhorn-backend:9500" - {{- if .Values.defaultSettings.registrySecret }} + {{- if .Values.privateRegistry.registrySecret }} imagePullSecrets: - - name: {{ .Values.defaultSettings.registrySecret }} + - name: {{ .Values.privateRegistry.registrySecret }} {{- end }} --- kind: Service @@ -45,7 +41,7 @@ metadata: kubernetes.io/cluster-service: "true" {{- end }} name: longhorn-frontend - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} spec: {{- if eq .Values.service.ui.type "Rancher-Proxy" }} type: ClusterIP diff --git a/charts/longhorn/templates/ingress.yaml b/charts/longhorn/templates/ingress.yaml index e3e9e3ee..5b3a4058 100644 --- a/charts/longhorn/templates/ingress.yaml +++ b/charts/longhorn/templates/ingress.yaml @@ -3,6 +3,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: longhorn-ingress + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn-ingress annotations: diff --git a/charts/longhorn/templates/postupgrade-job.yaml b/charts/longhorn/templates/postupgrade-job.yaml index 41287c79..6b6a3859 100644 --- a/charts/longhorn/templates/postupgrade-job.yaml +++ b/charts/longhorn/templates/postupgrade-job.yaml @@ -5,7 +5,7 @@ metadata: "helm.sh/hook": post-upgrade "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation name: longhorn-post-upgrade - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: activeDeadlineSeconds: 900 @@ -17,12 +17,8 @@ spec: spec: containers: - name: longhorn-post-upgrade - {{- if .Values.privateRegistry.registryUrl }} - image: "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- else }} - image: "{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- end }} - imagePullPolicy: Always + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: IfNotPresent command: - longhorn-manager - post-upgrade @@ -32,8 +28,8 @@ spec: fieldRef: fieldPath: metadata.namespace restartPolicy: OnFailure - {{- if .Values.defaultSettings.registrySecret }} + {{- if .Values.privateRegistry.registrySecret }} imagePullSecrets: - - name: {{ .Values.defaultSettings.registrySecret }} + - name: {{ .Values.privateRegistry.registrySecret }} {{- end }} serviceAccountName: longhorn-service-account diff --git a/charts/longhorn/templates/psp.yaml b/charts/longhorn/templates/psp.yaml index 66479b4f..a2dfc05b 100644 --- a/charts/longhorn/templates/psp.yaml +++ b/charts/longhorn/templates/psp.yaml @@ -35,7 +35,7 @@ kind: Role metadata: name: longhorn-psp-role labels: {{- include "longhorn.labels" . | nindent 4 }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} rules: - apiGroups: - policy @@ -51,7 +51,7 @@ kind: RoleBinding metadata: name: longhorn-psp-binding labels: {{- include "longhorn.labels" . | nindent 4 }} - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -59,8 +59,8 @@ roleRef: subjects: - kind: ServiceAccount name: longhorn-service-account - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} - kind: ServiceAccount name: default - namespace: {{ .Release.Namespace }} -{{- end }} \ No newline at end of file + namespace: {{ include "release_namespace" . }} +{{- end }} diff --git a/charts/longhorn/templates/registry-secret.yml b/charts/longhorn/templates/registry-secret.yml index eeb9a8f4..1c7565fe 100644 --- a/charts/longhorn/templates/registry-secret.yml +++ b/charts/longhorn/templates/registry-secret.yml @@ -1,10 +1,11 @@ -{{- if .Values.defaultSettings.registrySecret }} +{{- if .Values.privateRegistry.registrySecret }} apiVersion: v1 kind: Secret metadata: - name: {{ .Values.defaultSettings.registrySecret }} + name: {{ .Values.privateRegistry.registrySecret }} + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson data: .dockerconfigjson: {{ template "secret" . }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/longhorn/templates/serviceaccount.yaml b/charts/longhorn/templates/serviceaccount.yaml index a2280b44..ad576c35 100644 --- a/charts/longhorn/templates/serviceaccount.yaml +++ b/charts/longhorn/templates/serviceaccount.yaml @@ -2,5 +2,5 @@ apiVersion: v1 kind: ServiceAccount metadata: name: longhorn-service-account - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} diff --git a/charts/longhorn/templates/storageclass.yaml b/charts/longhorn/templates/storageclass.yaml index 3fee340f..dea6aafd 100644 --- a/charts/longhorn/templates/storageclass.yaml +++ b/charts/longhorn/templates/storageclass.yaml @@ -1,14 +1,26 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 +apiVersion: v1 +kind: ConfigMap metadata: - name: longhorn - annotations: - storageclass.kubernetes.io/is-default-class: {{ .Values.persistence.defaultClass | quote }} + name: longhorn-storageclass + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} -provisioner: driver.longhorn.io -allowVolumeExpansion: true -parameters: - numberOfReplicas: "{{ .Values.persistence.defaultClassReplicaCount }}" - staleReplicaTimeout: "30" - fromBackup: "" - baseImage: "" +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: {{ .Values.persistence.defaultClass | quote }} + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "{{ .Values.persistence.reclaimPolicy }}" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "{{ .Values.persistence.defaultClassReplicaCount }}" + staleReplicaTimeout: "30" + fromBackup: "" + baseImage: "" + {{- if .Values.persistence.recurringJobs.enable }} + recurringJobs: '{{ .Values.persistence.recurringJobs.jobList }}' + {{- end }} diff --git a/charts/longhorn/templates/tls-secrets.yaml b/charts/longhorn/templates/tls-secrets.yaml index 7a75df09..a7ebf13e 100644 --- a/charts/longhorn/templates/tls-secrets.yaml +++ b/charts/longhorn/templates/tls-secrets.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: Secret metadata: name: longhorn + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} app: longhorn type: kubernetes.io/tls diff --git a/charts/longhorn/templates/uninstall-job.yaml b/charts/longhorn/templates/uninstall-job.yaml index f5696dd0..e7e9f145 100644 --- a/charts/longhorn/templates/uninstall-job.yaml +++ b/charts/longhorn/templates/uninstall-job.yaml @@ -5,7 +5,7 @@ metadata: "helm.sh/hook": pre-delete "helm.sh/hook-delete-policy": hook-succeeded name: longhorn-uninstall - namespace: {{ .Release.Namespace }} + namespace: {{ include "release_namespace" . }} labels: {{- include "longhorn.labels" . | nindent 4 }} spec: activeDeadlineSeconds: 900 @@ -17,12 +17,8 @@ spec: spec: containers: - name: longhorn-uninstall - {{- if .Values.privateRegistry.registryUrl }} - image: "{{ .Values.privateRegistry.registryUrl }}/{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- else }} - image: "{{ .Values.image.longhorn.manager }}:{{ .Values.image.longhorn.managerTag }}" - {{- end }} - imagePullPolicy: Always + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: IfNotPresent command: - longhorn-manager - uninstall @@ -33,8 +29,8 @@ spec: fieldRef: fieldPath: metadata.namespace restartPolicy: OnFailure - {{- if .Values.defaultSettings.registrySecret }} + {{- if .Values.privateRegistry.registrySecret }} imagePullSecrets: - - name: {{ .Values.defaultSettings.registrySecret }} + - name: {{ .Values.privateRegistry.registrySecret }} {{- end }} serviceAccountName: longhorn-service-account diff --git a/charts/longhorn/values.yaml b/charts/longhorn/values.yaml index 95406785..d3345d4f 100644 --- a/charts/longhorn/values.yaml +++ b/charts/longhorn/values.yaml @@ -1,16 +1,43 @@ # Default values for longhorn. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +global: + cattle: + systemDefaultRegistry: "" + image: longhorn: - engine: longhornio/longhorn-engine - engineTag: v1.0.2 - manager: longhornio/longhorn-manager - managerTag: v1.0.2 - ui: longhornio/longhorn-ui - uiTag: v1.0.2 - instanceManager: longhornio/longhorn-instance-manager - instanceManagerTag: v1_20200514 + engine: + repository: longhornio/longhorn-engine + tag: v1.1.0 + manager: + repository: longhornio/longhorn-manager + tag: v1.1.0 + ui: + repository: longhornio/longhorn-ui + tag: v1.1.0 + instanceManager: + repository: longhornio/longhorn-instance-manager + tag: v1_20201216 + shareManager: + repository: longhornio/longhorn-share-manager + tag: v1_20201204 + csi: + attacher: + repository: longhornio/csi-attacher + tag: v2.2.1-lh1 + provisioner: + repository: longhornio/csi-provisioner + tag: v1.6.0-lh1 + nodeDriverRegistrar: + repository: longhornio/csi-node-driver-registrar + tag: v1.2.0-lh1 + resizer: + repository: longhornio/csi-resizer + tag: v0.5.1-lh1 + snapshotter: + repository: longhornio/csi-snapshotter + tag: v2.1.1-lh1 pullPolicy: IfNotPresent service: @@ -24,26 +51,25 @@ service: persistence: defaultClass: true defaultClassReplicaCount: 3 + reclaimPolicy: Delete + recurringJobs: + enable: false + jobList: [] csi: - attacherImage: longhornio/csi-attacher - attacherImageTag: v2.0.0 - provisionerImage: longhornio/csi-provisioner - provisionerImageTag: v1.4.0 - nodeDriverRegistrarImage: longhornio/csi-node-driver-registrar - nodeDriverRegistrarImageTag: v1.2.0 - resizerImage: longhornio/csi-resizer - resizerImageTag: v0.3.0 kubeletRootDir: ~ attacherReplicaCount: ~ provisionerReplicaCount: ~ resizerReplicaCount: ~ + snapshotterReplicaCount: ~ defaultSettings: backupTarget: ~ backupTargetCredentialSecret: ~ + allowRecurringJobWhileVolumeDetached: ~ createDefaultDiskLabeledNodes: ~ defaultDataPath: ~ + defaultDataLocality: ~ replicaSoftAntiAffinity: ~ storageOverProvisioningPercentage: ~ storageMinimalAvailablePercentage: ~ @@ -54,17 +80,26 @@ defaultSettings: backupstorePollInterval: ~ taintToleration: ~ priorityClass: ~ - registrySecret: ~ autoSalvage: ~ + autoDeletePodWhenVolumeDetachedUnexpectedly: ~ disableSchedulingOnCordonedNode: ~ replicaZoneSoftAntiAffinity: ~ volumeAttachmentRecoveryPolicy: ~ + nodeDownPodDeletionPolicy: ~ + allowNodeDrainWithLastHealthyReplica: ~ mkfsExt4Parameters: ~ + disableReplicaRebuild: ~ + replicaReplenishmentWaitInterval: ~ + disableRevisionCounter: ~ + systemManagedPodsImagePullPolicy: ~ + allowVolumeCreationWithDegradedAvailability: ~ + autoCleanupSystemGeneratedSnapshot: ~ privateRegistry: registryUrl: ~ registryUser: ~ registryPasswd: ~ + registrySecret: ~ resources: {} # We usually recommend not to specify default resources and to leave this as a conscious @@ -121,3 +156,7 @@ ingress: # Configure a pod security policy in the Longhorn namespace to allow privileged pods enablePSP: true + +## Specify override namespace, specifically this is useful for using longhorn as sub-chart +## and its release namespace is not the `longhorn-system` +namespaceOverride: ""