Skip to content

Commit

Permalink
Merge pull request #58 from galexrt/libvirt_support
Browse files Browse the repository at this point in the history 
Add libvirt support
  • Loading branch information
@galexrt
galexrt authored Feb 23, 2020
2 parents 8342a72 + a97a4a7 commit aa9152e
Show file tree
Hide file tree
Showing 9 changed files with 211 additions and 151 deletions.
51 changes: 39 additions & 12 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ REVERSE_LINES=sed -e '1!G;h;$$!d'
# === BEGIN USER OPTIONS ===
# Vagrantfile set to use.
BOX_OS ?= fedora
# Box setup
#BOX_IMAGE
# Vagrant Provider
VAGRANT_DEFAULT_PROVIDER ?= virtualbox
# Disk setup
DISK_COUNT ?= 1
DISK_SIZE_GB ?= 25
Expand All @@ -23,6 +23,8 @@ MASTER_IP ?= 192.168.26.10
NODE_IP_NW ?= 192.168.26.
POD_NW_CIDR ?= 10.244.0.0/16

KUBETOKEN =

KUBECTL_AUTO_CONF ?= true

# Kubernetes and kubeadm
Expand All @@ -39,6 +41,7 @@ K8S_DASHBOARD ?= false
K8S_DASHBOARD_VERSION ?= v1.10.1

CLUSTER_NAME ?= $(shell basename $(MFILECWD))
USER_SSHPUBKEY ?=

VAGRANT_LOG ?=
VAGRANT_VAGRANTFILE ?= $(MFILECWD)/vagrantfiles/Vagrantfile
Expand Down Expand Up @@ -79,16 +82,28 @@ versions: ## Print the "imporant" tools versions out for easier debugging.

@echo "Vagrant version:"
@vagrant --version
ifeq ($(VAGRANT_DEFAULT_PROVIDER), "virtualbox")
@echo "vboxmanage version:"
@vboxmanage --version
endif
ifeq ($(VAGRANT_DEFAULT_PROVIDER), "libvirt")
@echo "libvirtd version:
@libvirtd --version
endif

@echo "=== END Version Info ==="

up: preflight ## Start Kubernetes Vagrant multi-node cluster. Creates, starts and bootsup the master and node VMs.
@$(MAKE) start

start: preflight pull
ifeq ($(VAGRANT_DEFAULT_PROVIDER), "virtualbox")
@$(MAKE) start-master start-nodes
else
# Need to start master and nodes separately due to some weird IP assignment side effects
@$(MAKE) start-master
@$(MAKE) start-nodes
endif
@if $(KUBECTL_AUTO_CONF); then \
$(MAKE) kubectl; \
else \
Expand Down Expand Up @@ -146,26 +161,26 @@ kubectl-delete: ## Delete the created CLUSTER_NAME context from the kubeconfig (
$(eval CLUSTERCERTSDIR := $(shell mktemp -d))
if (kubectl config get-contexts $(CLUSTER_NAME) > /dev/null 2>&1); then kubectl config delete-context $(CLUSTER_NAME); fi

pull: ## Add and download, or update the box image on the host.
@if !(vagrant box list | grep -q $(shell grep "^\$$box_image.*=.*'.*'\.freeze" "$(MFILECWD)/vagrantfiles/$(BOX_OS)/common" | cut -d\' -f4)); then \
pull: ## Add and download, or update the box image for the chosen provider on the host.
@if !(vagrant box list | grep "$$(grep "^\$$box_image.*=.*'.*'\.freeze" "$(MFILECWD)/vagrantfiles/$(BOX_OS)/common" | cut -d\' -f4)" | grep -qi "$(VAGRANT_DEFAULT_PROVIDER)"); then \
vagrant \
box \
add \
--provider=virtualbox \
--provider $(VAGRANT_DEFAULT_PROVIDER) \
$(shell grep "^\$$box_image.*=.*'.*'\.freeze" "$(MFILECWD)/vagrantfiles/$(BOX_OS)/common" | cut -d\' -f4); \
else \
vagrant \
box \
update \
--provider=virtualbox \
--provider $(VAGRANT_DEFAULT_PROVIDER) \
--box=$(shell grep "^\$$box_image.*=.*'.*'\.freeze" "$(MFILECWD)/vagrantfiles/$(BOX_OS)/common" | cut -d\' -f4); \
fi

start-master: preflight ## Start up master VM (automatically done by `up` target).
vagrant up
vagrant up --provider $(VAGRANT_DEFAULT_PROVIDER)

start-node-%: preflight ## Start node VM, where `%` is the number of the node.
NODE=$* vagrant up
NODE=$* vagrant up --provider $(VAGRANT_DEFAULT_PROVIDER)

start-nodes: preflight $(shell for i in $(shell seq 1 $(NODE_COUNT)); do echo "start-node-$$i"; done) ## Create and start all node VMs by utilizing the `node-X` target (automatically done by `up` target).

Expand Down Expand Up @@ -197,11 +212,11 @@ clean-node-%: ## Remove a node VM, where `%` is the number of the node.
clean-nodes: $(shell for i in $(shell seq 1 $(NODE_COUNT)); do echo "clean-node-$$i"; done) ## Remove all node VMs.

clean-data: ## Remove data (shared folders) and disks of all VMs (master and nodes).
rm -v -rf "$(PWD)/data/"*
rm -v -rf "$(PWD)/.vagrant/KUBETOKEN"
rm -v -rf "$(MFILECWD)/data/"*
rm -v -rf "$(MFILECWD)/.vagrant/KUBETOKEN"

clean-force: ## Remove all drives which should normally have been removed by the normal clean-master or clean-node-% targets.
rm -v -rf "$(PWD)/.vagrant/"*.vdi
rm -v -rf "$(MFILECWD)/.vagrant/"*.vdi "$(MFILECWD)/.vagrant/"*.img

vagrant-reload: vagrant-reload-master vagrant-reload-nodes ## Run vagrant reload on master and nodes.

Expand Down Expand Up @@ -229,6 +244,16 @@ load-image-node-%: ## Load local/pulled image into node VM, where `%` is the num

load-image-nodes: $(shell for i in $(shell seq 1 $(NODE_COUNT)); do echo "load-image-node-$$i"; done) ## Load local/pulled Docker image into all node VMs.

ssh-config: ssh-config-master ssh-config-nodes ## Generate SSH config for master and nodes.

ssh-config-master: ## Generate SSH config just for the master.
@vagrant ssh-config --host "master"

ssh-config-nodes: $(shell for i in $(shell seq 1 $(NODE_COUNT)); do echo "ssh-config-$$i"; done) ## Generate SSH config just for the nodes.

ssh-config-node-%: $(shell for i in $(shell seq 1 $(NODE_COUNT)); do echo "ssh-config-$$i"; done) ## Generate SSH config just for the one node number given.
@NODE=$* vagrant ssh-config --host "master"

status: status-master $(shell for i in $(shell seq 1 $(NODE_COUNT)); do echo "status-node-$$i"; done) ## Show status of master and all node VMs.

status-master: ## Show status of the master VM.
Expand Down Expand Up @@ -259,4 +284,6 @@ help: ## Show this help menu.
.PHONY: clean clean-data clean-master clean-nodes help kubectl kubectl-delete \
load-image load-image-master load-image-nodes preflight ssh-master start-master \
start-nodes status-master status-nodes status stop-master stop-nodes \
vagrant-reload vagrant-reload-master vagrant-reload-nodes stop token up
vagrant-reload vagrant-reload-master vagrant-reload-nodes stop token up \
ssh-config ssh-config-master ssh-config-nodes

91 changes: 56 additions & 35 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,62 +11,69 @@ A demo of the start and destroy of a cluster can be found here: [README.md Demo
- [Quickstart](#quickstart)
- [Different OS / Vagrantfiles](#different-os--vagrantfiles)
- [Usage](#usage)
- [Starting the environment](#starting-the-environment)
- [Faster (parallel) environment start](#faster-parallel-environment-start)
- [Show status of VMs](#show-status-of-vms)
- [Shutting down the environment](#shutting-down-the-environment)
- [Copy local Docker image into VMs](#copy-local-docker-image-into-vms)
- [Data inside VM](#data-inside-vm)
- [Show `make` targets](#show-make-targets)
- [Starting the environment](#starting-the-environment)
- [Faster (parallel) environment start](#faster-parallel-environment-start)
- [Show status of VMs](#show-status-of-vms)
- [Shutting down the environment](#shutting-down-the-environment)
- [Copy local Docker image into VMs](#copy-local-docker-image-into-vms)
- [Data inside VM](#data-inside-vm)
- [Show `make` targets](#show-make-targets)
- [Variables](#variables)
- [Demo](#demo)
- [Start Cluster](#start-cluster)
- [Destroy Cluster](#destroy-cluster)
- [Start Cluster](#start-cluster)
- [Destroy Cluster](#destroy-cluster)

<!-- /TOC -->

## Prerequisites

* `make`
* `kubectl` - Optional when `KUBECTL_AUTO_CONF=false` (default `true`) is set.
* `kubectl` - Optional when `KUBECTL_AUTO_CONF` is set to `false` (default: `true`).
* `grep`
* `cut`
* `rsync`
* Source for randomness (only used to generate a kubeadm token, when no custom `KUBETOKEN` is given):
* `/dev/urandom`
* `openssl` command - Fallback for when `/dev/urandom` is not available.
* `/dev/urandom`
* `openssl` command - Fallback for when `/dev/urandom` is not available.
* Vagrant (>= `2.2.0`)
* Tested with `2.2.2` (if you should experience issues, please upgrade to at least this version or higher)
* Virtualbox
* Tested with `6.0.0` (if you should experience issues, please upgrade to at least this version or higher)
* `VBoxManage` binary in `PATH`.
* Tested with `2.2.2` (if you should experience issues, please upgrade to at least this Vagrant version or higher)
* Vagrant Provider (one of the following two is needed)
* Virtualbox
* Tested with `6.0.0` (if you should experience issues, please upgrade to at least this version or higher)
* `VBoxManage` binary in `PATH`.
* libvirt (`vagrant plugin install vagrant-libvirt`)
* Tested with `libvirtd` version `5.10.0`.
* Libvirt support is still a bit experimental and can be unstable (e.g., VMs not getting IPs).
* Troubleshooting: If your VM creation is hanging at `Waiting for domain to get an IP address...`, using `virsh` run `virsh force reset VM_NAME` (`VM_NAME` can be obtained using `virsh list` command) or in virt-manager `Force Reset` on the VM.

> **NOTE** `kubectl` is only needed when the `kubectl` auto configuration is enabled (default is enabled), to disable it set the variable `KUBECTL_AUTO_CONF` to `false`.
> For more information, see the [Variables](#variables) section.
## Hardware Requirements

* Master
* CPU: 2 Cores
* Memory: 2GB
* CPU: 2 Cores
* Memory: 2GB
* 1x Node:
* CPU: 2 Core
* Memory: 2GB
* CPU: 2 Core
* Memory: 2GB

These resources can be changed by setting the according variables for the `make up` command, see [Variables section](#variables),
These resources can be changed by setting the according variables for the `make up` command, see [Variables](#variables) section.

## Quickstart

To start with the defaults, 1x master and 2x workers, run the following:
```

```shell
$ make up -j 3
```

The `-j3` will cause three VMs to be started in parallel to speed up the cluster creation.
> **NOTE** Your `kubectl` is automatically configured to use a context for the
> created cluster, after the master VM is started.
> The context is named after the directory the `Makefile` is in.
```
```shell
$ kubectl config current-context
k8s-vagrant-multi-node
$ kubectl get componentstatus
Expand All @@ -76,9 +83,9 @@ controller-manager Healthy ok
etcd-0 Healthy {"health": "true"}
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
master Ready master 4m v1.10.4
node1 Ready <none> 4m v1.10.4
node2 Ready <none> 4m v1.10.4
master Ready master 4m v1.17.3
node1 Ready <none> 4m v1.17.3
node2 Ready <none> 4m v1.17.3
```

## Different OS / Vagrantfiles
Expand All @@ -100,26 +107,30 @@ To use a different set than the default `fedora` one's, add `BOX_OS=__NAME__` (w
### Starting the environment

To start up the Vagrant Kubernetes multi node environment with the default of two worker nodes + a master (not parallel) run:
```

```shell
$ make up
```

> **NOTE** Your `kubectl` is automatically configured to use a context for the
> created cluster, after the master VM is started.
> The context is named after the directory the `Makefile` is in.
### Faster (parallel) environment start

To start up 4 VMs in parallel run (`-j` flag does not control how many (worker) VMs are started, the `NODE_COUNT` variable is used for that):
```

```shell
$ NODE_COUNT=3 make up -j4
```

The flag `-j CORES/THREADS` allows yout to set how many VMs (Makefile targets) will be run at the same time.
You can also use `-j $(nproc)` to start as many VMs as cores/threads you have in your machine.
So to start up all VMs (master and three nodes) in parallel, you would add one to the chosen `NODE_COUNT`.

### Show status of VMs

```
```shell
$ make status
master not created (virtualbox)
node1 not created (virtualbox)
Expand All @@ -129,7 +140,8 @@ node2 not created (virtualbox)
### Shutting down the environment

To destroy the Vagrant environment run:
```

```shell
$ make clean
$ make clean-data
```
Expand All @@ -138,15 +150,17 @@ $ make clean-data

The `make load-image` target can be used to copy a docker image from your local docker daemon to all the VMs in your cluster.
The `IMG` variable can be expressed in a few ways, for example:
```

```shell
$ make load-image IMG=your_name/your_image_name:your_tag
$ make load-image IMG=your_name/your_image_name
$ make load-image IMG=my-private-registry.com/your_name/your_image_name:your_tag
```

You can also specify a new image name and tag to use after the image has been copied to the VM's by setting the `TAG` variable.
This will not change the image/tag in your local docker daemon, it will only affect the image in the VM's.
```

```shell
$ make load-image IMG=repo/image:tag TAG=new_repo/new_image:new_tag
```

Expand All @@ -156,7 +170,7 @@ See the `data/VM_NAME/` directories, where `VM_NAME` is for example `master`.

### Show `make` targets

```
```shell
$ make help
Usage: make [TARGET ...]

Expand All @@ -174,7 +188,11 @@ load-image-master Load local/pulled image into master VM.
load-image-node-% Load local/pulled image into node VM, where `%` is the number of the node.
load-image-nodes Load local/pulled Docker image into all node VMs.
preflight Run checks and gather variables, used for the the `up` target.
pull Add and download, or update the box image on the host.
pull Add and download, or update the box image for the chosen provider on the host.
ssh-config-master Generate SSH config just for the master.
ssh-config-node-% Generate SSH config just for the one node number given.
ssh-config-nodes Generate SSH config just for the nodes.
ssh-config Generate SSH config for master and nodes.
ssh-master SSH into the master VM.
ssh-node-% SSH into a node VM, where `%` is the number of the node.
start-master Start up master VM (automatically done by `up` target).
Expand Down Expand Up @@ -202,7 +220,8 @@ versions Print the "imporant" tools versions out for easie
| Variable Name | Default Value | Description |
| ------------------------------- | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `BOX_OS` | `fedora` | Which set of Vagrantfiles to use to start the VMs. |
| `BOX_IMAGE` | `` | Set the VM box image to use (only for override purposes). |
| `VAGRANT_DEFAULT_PROVIDER` | `virtualbox` | Which Vagrant provider to use. Available are `virtualbox` and `libvirt`. |
| `BOX_IMAGE` | `""` (empty) | Override the VM box image used (only use for override purposes as the image is set based on the `BOX_OS` variable). |
| `DISK_COUNT` | `1` | Set how many additional disks will be added to the VMs. |
| `DISK_SIZE_GB` | `25` GB | Size of additional disks added to the VMs. |
| `MASTER_CPUS` | `2` Core | Amount of cores to use for the master VM. |
Expand All @@ -224,6 +243,7 @@ versions Print the "imporant" tools versions out for easie
| `KUBE_PROXY_IPVS` | `false` | Enable IPVS kernel modules to then use IPVS for the kube-proxy. |
| `KUBE_NETWORK` | `flannel` | What CNI to install, if empty don't install any CNI. `flannel`, `canal` and `calico` are supported options. Ubuntu CNI is forced to use `canal` and can't be changed (see [Different OS / Vagrantfiles](#different-os--vagrantfiles)). |
| `KUBECTL_AUTO_CONF` | `true` | If `kubectl` should be automatically configured to be able to talk with the cluster (if disabled, removes need for `kubectl` binary). |
| `USER_SSHPUBKEY` | `""` (empty) | Your SSH **public key** (not private) to add to the VMs `vagrant` users `.ssh/authorized_keys` file during VM provisioning. |
## Demo
Expand All @@ -238,4 +258,5 @@ Please note that these terminal recordings are currently outdated.
[![asciicast](https://asciinema.org/a/186376.png)](https://asciinema.org/a/186376)
## Creating an Issue
Please attach the `make versions` output to the issue as is shown in the issue template. This makes debugging easier.
Loading

0 comments on commit aa9152e

Please sign in to comment.