feat: install new k3s cluster

k3s_cluster_v2/agents.tf

@@ -0,0 +1,22 @@
+resource "hcloud_server" "agents" {
+  count = var.compute_count
+  name  = "k3s-agent-${count.index}"
+
+  image       = data.hcloud_image.ubuntu.name
+  server_type = var.compute_server_type
+  location    = var.server_location
+
+  ssh_keys = [data.hcloud_ssh_key.default.id]
+  labels = {
+    provisioner = "terraform",
+    engine      = "k3s",
+    node_type   = "agent",
+  }
+}
+
+resource "hcloud_server_network" "agents_network" {
+  count     = length(hcloud_server.agents)
+  server_id = hcloud_server.agents[count.index].id
+  subnet_id = hcloud_network_subnet.k3s_nodes.id
+  ip        = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + var.control_count + count.index)
+}

k3s_cluster_v2/k3s.tf

@@ -0,0 +1,123 @@
+module "k3s" {
+  source = "xunleii/k3s/module"
+
+  depends_on_    = hcloud_server.agents
+  k3s_version    = var.install_k3s_version
+  cluster_domain = "cluster.local"
+  cidr = {
+    pods     = "10.42.0.0/16"
+    services = "10.43.0.0/16"
+  }
+  drain_timeout  = "30s"
+  managed_fields = ["label", "taint"] // ignore annotations
+
+  global_flags = [
+    "--kubelet-arg cloud-provider=external" // required to use https://github.com/hetznercloud/hcloud-cloud-controller-manager
+  ]
+
+  servers = {
+    for i in range(length(hcloud_server.control_planes)) :
+    hcloud_server.control_planes[i].name => {
+      ip = hcloud_server_network.control_planes[i].ip
+      connection = {
+        type = "ssh"
+        host = hcloud_server.control_planes[i].ipv4_address
+
+        agent = true
+      }
+      flags       = ["--disable-cloud-controller", "--tls-san ${var.domain}"]
+      annotations = { "server_id" : i } // theses annotations will not be managed by this module
+    }
+  }
+
+  agents = {
+    for i in range(length(hcloud_server.agents)) :
+    "${hcloud_server.agents[i].name}_node" => {
+      name = hcloud_server.agents[i].name
+      ip   = hcloud_server_network.agents_network[i].ip
+      connection = {
+        type = "ssh"
+        host = hcloud_server.agents[i].ipv4_address
+      }
+
+      labels = {}
+      taints = {}
+    }
+  }
+}
+
+provider "kubernetes" {
+  host                   = module.k3s.kubernetes.api_endpoint
+  cluster_ca_certificate = module.k3s.kubernetes.cluster_ca_certificate
+  client_certificate     = module.k3s.kubernetes.client_certificate
+  client_key             = module.k3s.kubernetes.client_key
+}
+
+resource "kubernetes_service_account" "bootstrap" {
+  depends_on = [module.k3s.kubernetes_ready]
+
+  metadata {
+    name      = "bootstrap"
+    namespace = "default"
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "boostrap" {
+  depends_on = [module.k3s.kubernetes_ready]
+
+  metadata {
+    name = "bootstrap"
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = "bootstrap"
+    namespace = "default"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "admin"
+  }
+}
+
+data "kubernetes_secret" "sa_credentials" {
+  metadata {
+    name      = kubernetes_service_account.bootstrap.default_secret_name
+    namespace = "default"
+  }
+}
+
+## hcloud-cloud-controller-manager is necessary for cluster bootstrap
+
+data "http" "hcloud_cloud_controller_manager" {
+  url = "https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/v1.12.1/deploy/ccm-networks.yaml"
+}
+
+locals {
+  hccm_all_manifests = split("---", data.http.hcloud_cloud_controller_manager.body)
+
+  // first element is only comment
+  hccm_actual_manifests = slice(local.hccm_all_manifests, 1, length(local.hccm_all_manifests))
+}
+
+resource "kubernetes_manifest" "hcloud_cloud_controller_manager" {
+  for_each = toset(
+    local.hccm_actual_manifests
+  )
+
+  manifest = yamldecode(each.key)
+}
+
+resource "kubernetes_secret" "hcloud_token" {
+  metadata {
+    name      = "hcloud"
+    namespace = "kube-system"
+  }
+
+  data = {
+    token   = var.hcloud_ccm_token
+    network = hcloud_network.k3s.id
+  }
+}

k3s_cluster_v2/main.tf

@@ -0,0 +1,46 @@
+data "hcloud_ssh_key" "default" {
+  name = "default"
+}
+
+resource "hcloud_network" "k3s" {
+  name     = "k3s-network"
+  ip_range = "10.0.0.0/8"
+}
+
+resource "hcloud_network_subnet" "k3s_nodes" {
+  type         = "cloud"
+  network_id   = hcloud_network.k3s.id
+  network_zone = "eu-central"
+  ip_range     = "10.254.1.0/24"
+}
+
+resource "hcloud_network_subnet" "lb" {
+  type         = "cloud"
+  network_id   = hcloud_network.k3s.id
+  network_zone = "eu-central"
+  ip_range     = "10.254.2.0/24"
+}
+
+data "hcloud_image" "ubuntu" {
+  name = var.server_image
+}
+
+### Loadbalancer
+
+resource "hcloud_load_balancer" "k3s" {
+  name               = "k3s"
+  load_balancer_type = var.load_balancer_type
+  location           = var.server_location
+}
+
+resource "hcloud_load_balancer_network" "k3s" {
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  subnet_id        = hcloud_network_subnet.lb.id
+}
+
+
+resource "hcloud_rdns" "k3s" {
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  ip_address       = hcloud_load_balancer.k3s.ipv4
+  dns_ptr          = var.domain
+}

k3s_cluster_v2/output.tf

@@ -0,0 +1,9 @@
+output "summary" {
+  value = module.k3s.summary
+}
+
+output "bootstrap_sa" {
+  description = "Bootstrap ServiceAccount. Can be used by Terraform to provision this cluster."
+  value       = data.kubernetes_secret.sa_credentials.data
+  sensitive   = true
+}

k3s_cluster_v2/server.tf

@@ -0,0 +1,38 @@
+resource "hcloud_server" "control_planes" {
+  count = var.control_count
+  name  = "k3s-control-plane-${count.index}"
+
+  image       = data.hcloud_image.ubuntu.name
+  server_type = var.control_server_type
+  location    = var.server_location
+
+  ssh_keys = [data.hcloud_ssh_key.default.id]
+  labels = {
+    provisioner = "terraform",
+    engine      = "k3s",
+    node_type   = "control-plane"
+  }
+}
+
+resource "hcloud_server_network" "control_planes" {
+  count     = var.control_count
+  subnet_id = hcloud_network_subnet.k3s_nodes.id
+  server_id = hcloud_server.control_planes[count.index].id
+  ip        = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + count.index)
+}
+
+# LB
+
+resource "hcloud_load_balancer_service" "api" {
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  protocol         = "tcp"
+  listen_port      = 6443
+  destination_port = 6443
+}
+
+resource "hcloud_load_balancer_target" "api" {
+  count            = var.control_count
+  type             = "server"
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  server_id        = hcloud_server.control_planes[count.index].id
+}

k3s_cluster_v2/variables.tf

@@ -0,0 +1,57 @@
+variable "name" {
+  type = string
+}
+
+variable "server_image" {
+  type = string
+  # With ubuntu-20.04 k3s crashes on start (v1.17.4+k3s1)
+  default = "ubuntu-18.04"
+}
+
+variable "server_location" {
+  type = string
+}
+
+variable "control_server_type" {
+  type    = string
+  default = "cx21"
+}
+
+variable "compute_server_type" {
+  type    = string
+  default = "cpx21"
+}
+
+variable "control_count" {
+  description = "Number of control plane nodes."
+  default     = 3
+}
+
+variable "compute_count" {
+  type    = number
+  default = 1
+}
+
+variable "load_balancer_type" {
+  type    = string
+  default = "lb11"
+}
+
+variable "domain" {
+  type = string
+}
+
+variable "install_k3s_version" {
+  type    = string
+  default = "v1.22.4+k3s1"
+}
+
+variable "ssh_key" {
+  description = "SSH public Key content needed to provision the instances."
+  type        = string
+}
+
+variable "hcloud_ccm_token" {
+  description = "HCloud API Token used by the hcloud-cloud-controller-manager"
+  type        = string
+}

k3s_cluster_v2/versions.tf

@@ -0,0 +1,22 @@
+
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    hcloud = {
+      source = "hetznercloud/hcloud"
+    }
+    tls = {
+      source = "hashicorp/tls"
+    }
+    template = {
+      source = "hashicorp/template"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+    null = {
+      source = "hashicorp/null"
+    }
+  }
+}