Integrate terraform-libvirt-provider support virtual machine provisioning

main.tf

@@ -79,6 +79,26 @@ module "provider" {
 #   ssh_keys        = var.upcloud_ssh_keys
 # }
 
+# module "provider" {
+#   source = "./provider/libvirt"
+# 
+#   disk_size_gb             = var.libvirt_disk_size_gb
+#   memory_mb                = var.libvirt_memory_mb
+#   vcpus                    = var.libvirt_vcpus
+#   image_source             = var.libvirt_image_source
+#   basename                 = var.libvirt_basename
+#   ssh_keys                 = var.libvirt_ssh_keys
+#   ssh_keys_github_username = var.libvirt_ssh_keys_github_username 
+#   do_package_upgrade       = var.libvirt_do_package_upgrade
+#   libvirt_connection_uri   = var.libvirt_connection_uri
+#   public_gateway           = var.libvirt_public_gateway
+#   public_nameserver        = var.libvirt_public_nameserver
+#   public_iprange           = var.libvirt_public_iprange
+#   public_iprange_offset    = var.libvirt_public_iprange_offset
+#   hosts                    = var.node_count
+#   hostname_format          = var.hostname_format
+# }
+
 module "swap" {
   source = "./service/swap"
 

provider/libvirt/README.md

@@ -0,0 +1,101 @@
+# libvirt Provisioner
+
+[Terraform provider for libvirt](https://github.com/dmacvicar/terraform-provider-libvirt) is a non-official Terraform provider to provision KVM/QEMU based virtual machines.
+This module integrates the libvirt provider with hobby-kube.
+
+## Prerequisites
+
+First of all, a working KVM / QEMU host with bridged networking is needed.
+The module works both when provisioned from the virtualization host as well as on a machine with proper `qemu+ssh`-access to a host machine.
+Some installations have to be made **on the machine that is used for Terraform provisioning**:
+
+* libvirt
+* terraform-provider-libvirt
+* mkisofs
+
+### Linux
+
+[Libvirt](https://libvirt.org/) needs to be installed to communicate with the Linux virtualization host, if the provisioning machine is not the host as well. 
+On Ubuntu this can be done with
+```
+sudo apt-get install libvirt-clients
+```
+
+[Download a suitable terraform-provider-libvirt release](https://github.com/dmacvicar/terraform-provider-libvirt/releases) and extract the binary.
+Once the binary is extracted, it can be placed in the user's Terraform plugins cache:
+
+```
+export TERRAFORM_LIBVIRT_VERSION="0.6.2"
+mkdir -p ~/.local/share/terraform/plugins/registry.terraform.io/dmacvicar/libvirt/$TERRAFORM_LIBVIRT_VERSION/linux_amd64
+cp terraform-provider-libvirt ~/.local/share/terraform/plugins/registry.terraform.io/dmacvicar/libvirt/$TERRAFORM_LIBVIRT_VERSION/linux_amd64
+```
+
+More information on how to install the provider for Terraform >= 0.13 can be found [here](https://github.com/dmacvicar/terraform-provider-libvirt/blob/master/docs/migration-13.md).
+
+`mkisofs` is also needed.
+On Ubuntu it can be installed with
+
+```
+sudo apt install genisoimage
+```
+
+### MacOS
+
+[Libvirt](https://libvirt.org/) needs to be installed to communicate with the Linux virtualization host. 
+This can be done with Homebrew:
+```
+brew install libvirt
+```
+
+There is no pre-compiled binary for terraform-provider-libvirt for MacOS.
+Instead, it can be [built from source](https://github.com/dmacvicar/terraform-provider-libvirt#building-from-source).
+
+Similar to Linux, the resulting binary needs to be placed in the [user's terraform plugin cache directory](https://www.terraform.io/docs/commands/cli-config.html#provider-installation):
+
+```
+export TERRAFORM_LIBVIRT_VERSION="0.6.2"
+mkdir -p ~/Library/Application\ Support/io.terraform/plugins/registry.terraform.io/dmacvicar/libvirt/$TERRAFORM_LIBVIRT_VERSION/darwin_amd64
+cp terraform-provider-libvirt ~/Library/Application\ Support/io.terraform/plugins/registry.terraform.io/dmacvicar/libvirt/$TERRAFORM_LIBVIRT_VERSION/darwin_amd64
+```
+
+`mkisofs` is also needed.
+It comes with `cdrtools`, which can be installed with Homebrew:
+
+```
+brew install cdrtools
+```
+
+## Usage and Configuration
+
+Uncomment the libvirt-provider module in the top-level `main.tf` file.
+Configure your desired setup in `terraform.tfvars`.
+
+See the provided `terraform.tfvars.example` file in the module directory for possible options and explanations.
+
+### Network Setup
+
+For now, only bridged networking is supported.
+Bridge `br0` must be provisioned on the virtualization host.
+
+Nodes will be provisioned with static IP Addresses within the given briged network.
+Given your bridged network is `192.168.100.0/24`, a minimim viable configuration with three hosts and IP addresses counted from `192.168.100.120` looks like this:
+
+```
+# Count of nodes to provide (must be provided)
+node_count                       = 3
+
+# Range of IPs in the bridged host network to use for provisioning
+# (must be provided)
+libvirt_public_iprange           = "192.168.100.0/24"
+
+# Offset in public ip range to start with when providing node IPs (default: 1)
+libvirt_public_iprange_offset    = 120
+
+# The gateway for the node to use
+# (must be provided)
+libvirt_public_gateway           = "192.168.100.1"
+
+# The nameserver for the node to use
+# (must be provided)
+libvirt_public_nameserver        = "192.168.100.1"
+```

provider/libvirt/main.tf

@@ -0,0 +1,232 @@
+variable "hosts" {
+  default = 0
+}
+
+variable "disk_size_gb" {
+  default = 50
+}
+
+variable "memory_mb" {
+  default = 1024
+}
+
+variable "vcpus" {
+  default = 1
+}
+
+variable "image_source" {
+  type = string
+  default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64.img"
+}
+
+variable "basename" {
+  type = string
+  default = "hobbykube"
+}
+
+variable "ssh_keys" {
+  type    = list
+  default = []
+}
+
+variable "ssh_keys_github_username" {
+  default = ""
+}
+
+variable "apt_packages" {
+  type    = list
+  default = [
+      "net-tools",
+  ]
+}
+
+variable "do_package_upgrade" {
+  type    = string
+  default = "false"
+}
+
+variable "hostname_format" {
+  type    = string
+  default = "kube%d"
+}
+
+variable "libvirt_connection_uri" {
+  type    = string
+  default = "qemu:///system"
+}
+
+variable "public_gateway" {
+  type    = string
+  default = ""
+}
+
+variable "public_nameserver" {
+  type    = string
+  default = ""
+}
+
+variable "public_iprange" {
+  type    = string
+  default = ""
+}
+
+variable "public_iprange_offset" {
+  default = 1
+}
+
+locals {
+  user_data = [
+    for n in range(var.hosts) : templatefile("${path.module}/templates/cloud_init.cfg", {
+      hostname   = element(data.template_file.hostnames.*.rendered, n)
+      keys       = var.ssh_keys_github_username == "" ? var.ssh_keys : data.github_user.ssh_keys_user.ssh_keys
+      do_upgrade = var.do_package_upgrade
+    })
+  ]
+}
+
+provider "libvirt" {
+  uri = var.libvirt_connection_uri
+}
+
+resource "libvirt_pool" "storage_pool" {
+  name = "${var.basename}-pool"
+  type = "dir"
+  path = "/var/lib/libvirt/images/${var.basename}-pool"
+}
+
+resource "libvirt_volume" "os_image" {
+  name = "${var.basename}-baseimage-ubuntu-qcow2"
+  pool = libvirt_pool.storage_pool.name
+  source = var.image_source
+  format = "qcow2"
+}
+
+resource "libvirt_volume" "volume" {
+  count          = var.hosts
+  name           = "${var.basename}-volume-${element(data.template_file.hostnames.*.rendered, count.index)}"
+  pool           = libvirt_pool.storage_pool.name
+  base_volume_id = libvirt_volume.os_image.id
+  size           = var.disk_size_gb * 1024 * 1024 * 1024
+}
+
+resource "libvirt_cloudinit_disk" "commoninit" {
+  count          = var.hosts
+  name           = "${element(data.template_file.hostnames.*.rendered, count.index)}-commoninit.iso"
+  user_data      = element(local.user_data, count.index)
+  network_config = element(data.template_file.network_config.*.rendered, count.index)
+  pool           = libvirt_pool.storage_pool.name
+}
+
+resource "libvirt_domain" "node_domain" {
+  count      = var.hosts
+  name       = "${var.basename}-${element(data.template_file.hostnames.*.rendered, count.index)}"
+  memory     = var.memory_mb
+  vcpu       = var.vcpus
+
+  cloudinit  = element(libvirt_cloudinit_disk.commoninit.*.id, count.index)
+
+  network_interface {
+    bridge = "br0"
+  }
+
+  # IMPORTANT: this is a known bug on cloud images, since they expect a console
+  # we need to pass it
+  # https://bugs.launchpad.net/cloud-images/+bug/1573095
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+
+  console {
+    type        = "pty"
+    target_type = "virtio"
+    target_port = "1"
+  }
+
+  disk {
+    volume_id = element(libvirt_volume.volume.*.id, count.index)
+  }
+
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+
+  connection {
+    user    = "root"
+    type    = "ssh"
+    timeout = "2m"
+    host    = element(data.template_file.public_ips.*.rendered, count.index)
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "cloud-init status --wait  > /dev/null 2>&1",
+      "DEBIAN_FRONTEND=noninteractive apt-get install -yq ufw ${join(" ", var.apt_packages)}",
+    ]
+  }
+
+}
+
+data "github_user" "ssh_keys_user" {
+  username = var.ssh_keys_github_username == "" ? "github" : var.ssh_keys_github_username
+}
+
+data "template_file" "public_ips" {
+  count    = var.hosts
+  template = "$${ip}"
+
+  vars = {
+    ip = cidrhost(var.public_iprange, count.index + var.public_iprange_offset)
+  }
+}
+
+data "template_file" "hostnames" {
+  count    = var.hosts
+  template = "$${name}"
+
+  vars = {
+    name = format(var.hostname_format, count.index + 1)
+  }
+}
+
+data "template_file" "network_config" {
+  count    = var.hosts
+  template = file("${path.module}/templates/network_config.cfg")
+
+  vars = {
+    address    = element(data.template_file.public_ips.*.rendered, count.index)
+    gateway    = var.public_gateway
+    nameserver = var.public_nameserver
+  }
+}
+
+output "hostnames" {
+  value = "${data.template_file.hostnames.*.rendered}"
+  depends_on     = [
+    libvirt_domain.node_domain,
+  ]
+}
+
+output "public_ips" {
+  value = "${data.template_file.public_ips.*.rendered}"
+  depends_on     = [
+    libvirt_domain.node_domain,
+  ]
+}
+
+output "private_ips" {
+  value = "${data.template_file.public_ips.*.rendered}"
+  depends_on     = [
+    libvirt_domain.node_domain,
+  ]
+}
+
+output "private_network_interface" {
+  value = "ens3"
+  depends_on     = [
+    libvirt_domain.node_domain,
+  ]
+}

provider/libvirt/templates/cloud_init.cfg

@@ -0,0 +1,22 @@
+#cloud-config
+hostname: ${hostname}
+
+package_update: true
+package_upgrade: ${do_upgrade}
+
+growpart:
+    mode: auto
+    devices: ["/"]
+    ignore_growroot_disabled: false
+
+ssh_pwauth: True
+
+users:
+- name: root
+  ssh_authorized_keys:
+%{ for key in keys }
+    - ${key}
+%{ endfor ~}
+
+packages:
+  - qemu-guest-agent

provider/libvirt/templates/network_config.cfg

@@ -0,0 +1,11 @@
+version: 2
+ethernets:
+  ens3:
+    dhcp4: no
+    dhcp6: no
+    addresses: 
+        - ${address}/24
+    gateway4: ${gateway}
+    nameservers:
+        addresses: 
+            - ${nameserver}