From 80751437c1668d2c347199bd758d40b849c3efba Mon Sep 17 00:00:00 2001 From: gunjan5 Date: Wed, 16 Aug 2017 21:08:03 -0700 Subject: [PATCH] Calico v2.5 TPR->CRD migration and upgrade k8s job and docs --- upgrade/v2.5/Dockerfile | 16 +++ upgrade/v2.5/Makefile | 14 +++ upgrade/v2.5/README.md | 94 ++++++++++++++++ upgrade/v2.5/manifests/crds.yaml | 61 +++++++++++ upgrade/v2.5/manifests/globalbgpconfig.yaml | 13 +++ upgrade/v2.5/manifests/tprs.yaml | 38 +++++++ upgrade/v2.5/manifests/upgrade-job.yaml | 106 ++++++++++++++++++ upgrade/v2.5/script/upgrade.sh | 113 ++++++++++++++++++++ 8 files changed, 455 insertions(+) create mode 100644 upgrade/v2.5/Dockerfile create mode 100644 upgrade/v2.5/Makefile create mode 100644 upgrade/v2.5/README.md create mode 100644 upgrade/v2.5/manifests/crds.yaml create mode 100644 upgrade/v2.5/manifests/globalbgpconfig.yaml create mode 100644 upgrade/v2.5/manifests/tprs.yaml create mode 100644 upgrade/v2.5/manifests/upgrade-job.yaml create mode 100755 upgrade/v2.5/script/upgrade.sh diff --git a/upgrade/v2.5/Dockerfile b/upgrade/v2.5/Dockerfile new file mode 100644 index 00000000000..9df0ddd5498 --- /dev/null +++ b/upgrade/v2.5/Dockerfile @@ -0,0 +1,16 @@ +FROM busybox + +MAINTAINER Gunjan Patel + +ADD dist/calicoctl-v1.4 /sbin/calicoctl-v1.4 +ADD dist/calicoctl-v1.5 /sbin/calicoctl-v1.5 +ADD dist/kubectl /sbin/kubectl + +ADD script/upgrade.sh /upgrade.sh +ADD manifests/crds.yaml /crds.yaml +ADD manifests/tprs.yaml /tprs.yaml +ADD manifests/globalbgpconfig.yaml /globalbgpconfig.yaml + +WORKDIR / + +CMD ["/upgrade.sh"] \ No newline at end of file diff --git a/upgrade/v2.5/Makefile b/upgrade/v2.5/Makefile new file mode 100644 index 00000000000..965b7644eb4 --- /dev/null +++ b/upgrade/v2.5/Makefile @@ -0,0 +1,14 @@ +dist/clis: + mkdir -p dist + curl -sSf -L https://github.com/projectcalico/calicoctl/releases/download/v1.4.1/calicoctl -o dist/calicoctl-v1.4 + curl -sSf -L https://github.com/projectcalico/calicoctl/releases/download/v1.5.0/calicoctl -o dist/calicoctl-v1.5 + curl -sSf -L https://storage.googleapis.com/kubernetes-release/release/v1.7.4/bin/linux/amd64/kubectl -o dist/kubectl + chmod +x dist/calicoctl-v1.4 + chmod +x dist/calicoctl-v1.5 + chmod +x dist/kubectl + +build-container: dist/clis + docker build -t calico/v2.5-upgrade:v0.0.1 . + +clean: + rm -rf dist diff --git a/upgrade/v2.5/README.md b/upgrade/v2.5/README.md new file mode 100644 index 00000000000..4739e2a3e9c --- /dev/null +++ b/upgrade/v2.5/README.md @@ -0,0 +1,94 @@ +# Calico v2.5.0 Upgrade Procedure + +## Why this is required + +Kubernetes v1.7 introduces a new API data type called CustomResourceDefinition (CRD) which will replace the alpha ThirdPartyResource (TPR). +Calico relies on TPR to store it's config data when it uses kubernetes datastore backend. Moving forward from kubernetes v1.8, +Kubernetes will remove the support for TPR in favor of CRD. In order to preserve the config data backed by TPR, and continue to +work with the future Kubernetes versions, it is required to go through this migration process. +See [this blog post](https://coreos.com/blog/custom-resource-kubernetes-v17) for more information on this new data type. + +## Steps required + +At a high level we need to do the following steps to make sure we have a successful upgrade. + + 1. Check to make sure if you need to do the migration + 2. Backup your TPR data (Data is in TPR right now) + 3. Run the migration job (Data is copied from TPR to CRD) + 4. Verify the migration (Data is still in both TPR and CRD, Calico is still using TPR) + 5. Upgrade Calico version to `v2.5.0` (Data is still in TPR and CRD, but Calico now uses CRD backed data) + 6. Verify Calico policy and networking works as expected + 7. Delete the TPRs + +## 1. Before you begin + +### 1.1 Do I need to go through this migration process? + +This is only required if you meet ALL of the following criteria: + +- Running Calico with Kubernetes +- Current Kubernetes version is `v1.7.x` +- Current Calico version is `v2.4.x` or lower +- Upgrading Calico to `v2.5.x` +- Calico is running with Kubernetes datastore backend (By running `calicoctl version` and making sure `Cluster Type` is `KDD`) + +### 1.2 Backup your config data + +We highly recommend backing up your config data before proceeding with the migration process. +We only need to backup the config backed by TPR resources; in the event the migration needs to be halted, you'll be able to restore the original data. + +> Note: the migration job does not delete your old data, so your config data backed by TPR will still be there until + deleted manually (explained in the last step of this doc). + +Run the following commands to backup your config data: + +> Note: use `calicoctl` version [v1.4.1](https://github.com/projectcalico/calicoctl/releases/tag/v1.4.1) + and `kubectl` version [v1.7.4](https://kubernetes.io/docs/tasks/tools/install-kubectl/) to backup the data. + +Since we will need `calicoctl` versions [v1.4.1](https://github.com/projectcalico/calicoctl/releases/tag/v1.4.1) and [v1.5.0](https://github.com/projectcalico/calicoctl/releases/tag/v1.5.0) for this upgrade, + we recommend downloading them both and suffixing the binaries with their respective versions (you can check the version by running `calicoctl version`) + + 1.2.1. `calicoctl_v1.4 get ippools -o yaml > ippool.yaml` + + 1.2.2. `calicoctl_v1.4 get bgppeers -o yaml > bgppeer.yaml` + + 1.2.3. `kubectl get globalconfig --all-namespaces -o yaml > tpr-felixconfig.yaml` + + 1.2.4. `kubectl get globalbgpconfig --all-namespaces -o yaml > tpr-bgpconfig.yaml` + +> Note: you may not have some of these resources if you're using Calico in policy-only mode. + +## 2. Migration process + + 2.1. Create the migration kubernetes job for your cluster: `kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/master/upgrade/v2.5/manifests/upgrade-job.yaml` + + 2.2. Check the job status `kubectl describe job/calico-upgrade-v2.5` + + 2.3. Check the upgrade job logs `kubectl logs ` to make sure there aren't any errors (You can get the upgrade job's pod name from the previous command output 'Message' field) + +## 3. After the migration + +### 3.1 Verify that the data has been copied correctly + +Make sure you have all the `IPPools`, `BGPPeers`, `GlobalFelixConfig` and `GlobalBGPConfig` resource configs you had before the upgrade: + + 3.1.1. `calicoctl_v1.5 get ippools -o wide` + + 3.1.2. `calicoctl_v1.5 get bgppeers -o wide` (This is only if you're running Calico BGP networking) + + 3.1.3. `kubectl get globalfelixconfigs.crd.projectcalico.org -o wide` + + 3.1.4. `kubectl get globalbgpconfigs.crd.projectcalico.org -o wide` + +## 4. Upgrade calico + + 4.1. (If you have RBAC enabled) Apply the updated RBAC manifest `kubectl apply -f https://docs.projectcalico.org/v2.5/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml` (this will revoke access to TPRs from calico-node) + + 4.2. Now you can upgrade Calico version to `v2.5.0` in your kubernetes Calico DaemonSet. (Make sure you reboot your calico-node pods one at a time if calico-node `updateStrategy` is not set to `RollingUpdate`) + + 4.3. Verify that everything is working as expected. + +## 5. Delete the old TPRs + +You can now delete your TPRs by running the following command: +`kubectl delete -f https://raw.githubusercontent.com/projectcalico/calico/master/upgrade/v2.5/manifests/tprs.yaml` diff --git a/upgrade/v2.5/manifests/crds.yaml b/upgrade/v2.5/manifests/crds.yaml new file mode 100644 index 00000000000..079940f8e9d --- /dev/null +++ b/upgrade/v2.5/manifests/crds.yaml @@ -0,0 +1,61 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Felix Configuration +kind: CustomResourceDefinition +metadata: + name: globalfelixconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalFelixConfig + plural: globalfelixconfigs + singular: globalfelixconfig + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico BGP Peers +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico IP Pools +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Network Policies +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy diff --git a/upgrade/v2.5/manifests/globalbgpconfig.yaml b/upgrade/v2.5/manifests/globalbgpconfig.yaml new file mode 100644 index 00000000000..f33f69b2f39 --- /dev/null +++ b/upgrade/v2.5/manifests/globalbgpconfig.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global BGP Configuration +kind: CustomResourceDefinition +metadata: + name: globalbgpconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalBGPConfig + plural: globalbgpconfigs + singular: globalbgpconfig diff --git a/upgrade/v2.5/manifests/tprs.yaml b/upgrade/v2.5/manifests/tprs.yaml new file mode 100644 index 00000000000..dca355f2ed1 --- /dev/null +++ b/upgrade/v2.5/manifests/tprs.yaml @@ -0,0 +1,38 @@ + +apiVersion: extensions/v1beta1 +description: Calico Global Configuration +kind: ThirdPartyResource +metadata: + name: global-config.projectcalico.org +versions: +- name: v1 + +--- + +apiVersion: extensions/v1beta1 +description: Calico IP Pools +kind: ThirdPartyResource +metadata: + name: ip-pool.projectcalico.org +versions: +- name: v1 + +--- + +apiVersion: extensions/v1beta1 +description: Calico Global BGP Configuration +kind: ThirdPartyResource +metadata: + name: global-bgp-config.projectcalico.org +versions: +- name: v1 + +--- + +apiVersion: extensions/v1beta1 +description: Calico Global BGP Peers +kind: ThirdPartyResource +metadata: + name: global-bgp-peer.projectcalico.org +versions: +- name: v1 diff --git a/upgrade/v2.5/manifests/upgrade-job.yaml b/upgrade/v2.5/manifests/upgrade-job.yaml new file mode 100644 index 00000000000..82314d5d9bb --- /dev/null +++ b/upgrade/v2.5/manifests/upgrade-job.yaml @@ -0,0 +1,106 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-25-migration +rules: + - apiGroups: [""] + resources: + - nodes + verbs: + - create + - get + - list + - post + - patch + - update + - watch + - apiGroups: ["extensions"] + resources: + - thirdpartyresources + verbs: + - create + - get + - list + - post + - patch + - update + - watch + - apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - post + - patch + - update + - watch + - apiGroups: ["projectcalico.org"] + resources: + - globalbgppeers + - globalconfigs + - globalbgpconfigs + - ippools + verbs: + - create + - get + - list + - post + - patch + - update + - watch + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - bgppeers + - globalbgpconfigs + - ippools + - globalnetworkpolicies + verbs: + - create + - get + - list + - post + - patch + - update + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-25-migration +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-25-migration +subjects: + - kind: ServiceAccount + name: calico-25-migration + namespace: default + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-25-migration + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: calico-upgrade-v2.5 +spec: + template: + metadata: + name: calico-upgrade-v2.5 + spec: + serviceAccountName: calico-25-migration + containers: + - name: calico-upgrade + image: gunjan5/upgrade:v2.5.16 + restartPolicy: Never diff --git a/upgrade/v2.5/script/upgrade.sh b/upgrade/v2.5/script/upgrade.sh new file mode 100755 index 00000000000..4432e910cbf --- /dev/null +++ b/upgrade/v2.5/script/upgrade.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +# Set the ENV var for KDD backend +export DATASTORE_TYPE=kubernetes + +# Create the CRDs by applying the crds.yaml manifest. +echo "Creating Calico CRDs..."; echo +/sbin/kubectl apply -f crds.yaml +# Applying TPRs with trps.yaml manifest. +/sbin/kubectl apply -f tprs.yaml + +# Get IPPools list with old calicoctl (from TPR) and save it as a yaml file. +# Apply the yaml file with the new calicoctl (to CRD) +echo "Migrating IPPools..." +/sbin/calicoctl-v1.4 get ippool -o yaml | tee ippool.yaml +if [ $? != 0 ]; then + echo "Failed to get IPPools throught calicoctl" + exit 1 +else if [ `cat ippool.yaml | wc -l` == 1 ]; then + echo "No IPPools found to migrate." + echo "Moving on..." +else + /sbin/calicoctl-v1.5 apply -f ippool.yaml + if [ $? != 0 ]; then + echo "Failed to apply the IPPools" + exit 1 + else + echo "Migrated the IPPools!" + fi + fi +fi + +# Get BGPPeers list with old calicoctl (from TPR) and save it as a yaml file. +# Apply the yaml file with the new calicoctl (to CRD) +echo; echo "Migrating BGPPeers..." +/sbin/calicoctl-v1.4 get bgppeers -o yaml | tee bgppeers.yaml +if [ $? != 0 ]; then + echo "Failed to get Global BGP Peers through calicoctl" + exit 1 +else if [ `cat bgppeers.yaml | wc -l` == 1 ]; then + echo "No BGPPeers found to migrate." + echo "Moving on..." +else + /sbin/calicoctl-v1.5 apply -f bgppeers.yaml + if [ $? != 0 ]; then + echo "Failed to apply the BGPPeers" + exit 1 + else + echo "Migrated the BGPPeers!" + fi + fi +fi + +# List all the Felix configs using kubectl and save it in a yaml file. +# Change the apiVersion from 'projectcalico.org/v1' to 'crd.projectcalico.org/v1', +# rename resource kind from 'GlobalConfig' to 'GlobalFelixConfig' and save it in a new yaml file. +# Apply the modified yaml file. +echo; echo "Migrating GlobalFelixConfig..." +/sbin/kubectl get globalconfig --all-namespaces -o yaml | tee tpr-felixconfig.yaml +if [ $? != 0 ]; then + echo "Failed to get Global Felix config through Kubectl" + exit 1 +else + grep 'items\: \[\]' tpr-felixconfig.yaml 1>/dev/null + if [ $? == 0 ]; then + echo "No GlobalFelixConfig found to migrate." + echo "Moving on..." + else + cat tpr-felixconfig.yaml | sed '/apiVersion/s/projectcalico\.org\/v1/crd\.projectcalico\.org\/v1/g' | sed '/kind/s/GlobalConfig/GlobalFelixConfig/g' | sed -e '/^\s*creationTimestamp/d' -e '/^\s*uid/d' -e '/^\s*resourceVersion/d' -e '/^\s*namespace/d' -e '/^\s*selfLink/d' > crd-felixconfig.yaml + echo; echo "Converted global-config.projectcalico.org/v1 to globalfelixconfigs.crd.projectcalico.org/v1" + cat crd-felixconfig.yaml + /sbin/kubectl apply -f crd-felixconfig.yaml + if [ $? != 0 ]; then + echo "Failed to apply the GlobalFelixConfig" + exit 1 + else + echo "Migrated the GlobalFelixConfig!" + fi + fi +fi + + +# List all the BGP configs using kubectl and save it in a yaml file. +# Change the apiVersion from 'projectcalico.org/v1' to 'crd.projectcalico.org/v1' and save it in a new yaml file. +# Apply the modified yaml file. +echo; echo "Migrating GlobalBGPConfig..." +/sbin/kubectl get globalbgpconfig --all-namespaces -o yaml | tee tpr-bgpconfig.yaml +if [ $? != 0 ]; then + echo "Failed to get the Global BGP config through Kubectl" + exit 1 +else + grep 'items\: \[\]' tpr-bgpconfig.yaml 1>/dev/null + if [ $? == 0 ]; then + echo "No GlobalBGPConfig found to migrate." + /sbin/kubectl apply -f globalbgpconfig.yaml + echo "Moving on..." + else + /sbin/kubectl apply -f globalbgpconfig.yaml + cat tpr-bgpconfig.yaml | sed -e '/apiVersion/s/projectcalico\.org\/v1/crd\.projectcalico\.org\/v1/g' -e '/kind/s/GlobalBgpConfig/GlobalBGPConfig/g' | sed -e '/^\s*creationTimestamp/d' -e '/^\s*uid/d' -e '/^\s*resourceVersion/d' -e '/^\s*namespace/d' -e '/^\s*selfLink/d' > crd-bgpconfig.yaml + echo; echo "Converted global-bgp-config.projectcalico.org/v1 to globalbgpconfigs.crd.projectcalico.org/v1" + cat crd-bgpconfig.yaml + /sbin/kubectl apply -f crd-bgpconfig.yaml + if [ $? != 0 ]; then + echo "Failed to apply the GlobalBGPConfig" + /sbin/kubectl delete -f globalbgpconfig.yaml + exit 1 + else + echo "Migrated the GlobalBGPConfig!" + fi + fi +fi + +echo; echo "Successfully migrated Calico data!" \ No newline at end of file