feat: install new k3s cluster

.terraform.lock.hcl

@@ -1,39 +1,79 @@
 # This file is maintained automatically by "terraform init".
 # Manual edits may be lost in future updates.
 
-provider "registry.terraform.io/hashicorp/null" {
-  version = "3.1.0"
+provider "registry.terraform.io/hashicorp/http" {
+  version     = "1.2.0"
+  constraints = "~> 1.2"
   hashes = [
-    "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=",
-    "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2",
-    "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515",
-    "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521",
-    "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2",
-    "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e",
-    "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53",
-    "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d",
-    "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8",
-    "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70",
-    "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b",
-    "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e",
+    "h1:g5/qN61yBHRqpGVm08BnBp1O3vVf1eIpMglYvsY13XE=",
+    "zh:12df09a69dbf8219d9544466928abc65cc8cf38a26258a7586921e1e791a89a2",
+    "zh:3a1a57be455011ae09df92939a2e4ea1ac1d5fd74b4617889b10a03e83b0238d",
+    "zh:48f380c5e9c3abb69b7ecc40ed68f24c9ad42727d58ad530cc61f2d0524d2664",
+    "zh:4d3102f731915609dd5dda55d1525c3ac3be20fae37feba2f0e2770c6e71c5a3",
+    "zh:5b8c1bbe08b9f0e5f9c0d498c4f0316807595a6d55b8c194fa36d692e66f8afe",
+    "zh:8d3a13bbf852a884905dce915665d8ee817fd436d088226ed4fabbc6db57d0fe",
+    "zh:9025dfd565785c9424243bf78546c9c1ff3064b732ad178d9142831f5c5bea66",
+    "zh:a370ac52da85934af38b60858be3e91f44a6a454b007f607b1b8a0ea307e2d27",
+    "zh:ab9a1be3a3c0c760d33d0a7234909f92282f704c8145443089abbf0fddda2d85",
+    "zh:d97ca45677d77eb49864887077e8457f518bd789f0609a19796b2fce70d1aadd",
+    "zh:ec2f2762eeabbd2706b26f1cb03aea5301f32468147d81cc84c823ef927bc292",
+    "zh:ef6851bde213957d5331039d9dc88c89d2d3182793387dae15419343c853664a",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/kubernetes" {
+  version = "2.7.0"
+  hashes = [
+    "h1:Cd7Kh8JmqhevGx+OsZ33QSa5hBHOs7CT9sfqTR5NEGM=",
+    "zh:0ee4ed19d951cf94a1b27bf813827bb5c51b48ac53efe827e77aa077e9d66021",
+    "zh:134ae78a19a43f94c2958e87ee8c6c1ae4593c145569bf71bf58a357882c18af",
+    "zh:191345c9c0808499378d6ac33237f41d4509769b06367630acbf1586e0fe1540",
+    "zh:1c058459ca1705dd97646d06c02df648a19a12789024a16155209bbdd75d03d6",
+    "zh:33ac31a72649b85962cf3404edd08e917704b225eb4064d63cfee5704635da56",
+    "zh:3dcc51d76711ff58d77a42811d8f65834fdb3632d44f56f2fe58eaca07c83a8a",
+    "zh:5d5ea922c72d2eb4abc67d9a34fcf92b3af33049325bb7384e772a9db90120dd",
+    "zh:9b02cda7aad45408fa2acbc844505d6a72c3998eb28e3568c1d35e142acd1ffd",
+    "zh:b7f4652ec526bc3e7a846a40b136dc4d1d873f643329056eeb8b7ac5dfb4e11e",
+    "zh:c8d1617426ca3d91bc0c5188f353dbca55ca777c28830e9bb0b79baef227885e",
+    "zh:ed7064a03e15ca6b2f026064f17fa3956775e34a930e5265399ffa7f6ef22ded",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/null" {
+  version     = "2.1.2"
+  constraints = "~> 2.1"
+  hashes = [
+    "h1:CFnENdqQu4g3LJNevA32aDxcUz2qGkRGQpFfkI8TCdE=",
+    "zh:0cc7236c1fbf971b8bad1540a7d0c5ac4579248239fd1034c023b0b660a8d1d5",
+    "zh:16fc2d9b10cf9e5123bf956e7032c338cc93a03be1ca2e9f3d3b7014c0e866c7",
+    "zh:1e26465ff40ded59cef1a9e745752eef9744471e69094d12f8bc40a060e8cdb9",
+    "zh:3c7afd28076245f455d4348af6c124b73409722be4a73680d4e4709a4f25ea91",
+    "zh:4190a92567efaa444527f19b28d65fac1a01aeba907013b5dceacd9ba2a23709",
+    "zh:677963437316b088fc1aac70fe7d608d2917c46530c4a25ec86a43e9acaf2811",
+    "zh:69fe15f6b851ff82700bc749c50a9299770515617e677c18cba2cb697daaff36",
+    "zh:6b505cc60cc1494e1cab61590bca127b06dd894d0b2a7dcacd23862bce1f492b",
+    "zh:719aa11ad7be974085af595089478eb24d5a021bc7b08364fa6745d9eb473dac",
+    "zh:7375b02189e14efbfaab994cd05d81e3ff8e46041fae778598b3903114093a3e",
+    "zh:c406573b2084a08f8faa0451923fbeb1ca6c5a5598bf039589ec2db13aacc622",
+    "zh:fb11299a3b20711595713d126abbbe78c554eb5995b02db536e9253686798fb6",
   ]
 }
 
 provider "registry.terraform.io/hashicorp/random" {
-  version = "3.1.0"
+  version     = "2.3.1"
+  constraints = "~> 2.2"
   hashes = [
-    "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=",
-    "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc",
-    "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626",
-    "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff",
-    "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2",
-    "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992",
-    "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427",
-    "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc",
-    "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f",
-    "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b",
-    "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7",
-    "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
+    "h1:bPBDLMpQzOjKhDlP9uH2UPIz9tSjcbCtLdiJ5ASmCx4=",
+    "zh:322ec2b56765162c193d0ff0f7634351bf4ca09d719558df82b92d31059995a1",
+    "zh:34c500857d14148bc6f7f21f9fc0d832ba31d00e1b911b0437d99dda29aeb108",
+    "zh:a3372514add474fcb07d2026a49b0c0d8a699e365f540e43e5f590eb924b95f7",
+    "zh:a54acd50f13f38a86454acad16b3487701608188fd2cc454fd48beee00127ae5",
+    "zh:b1285c47ab01fc11b6086f080da7d17b9155b3fac6a4175948a2abb5d52c60af",
+    "zh:c7e7bd46218cf504ca31d8a1e4eab5cad625c9951fb89d4f56861d5bddec9afb",
+    "zh:d9f95764480d7b884db7247a5ee9d1c72aaf1c1cde3d7b5e34bc33c4ef5ccf48",
+    "zh:ddf11fb807d61ff93c08206733129bf9668fc0c4adedf8a6ac38a7c382a24b35",
+    "zh:f631a0c1ffa94991d87cd5fe4e08c0dd6d36c780585f35a582c05ee2affb7e86",
+    "zh:fb5f5fb19e8a9ff73dbdee85a97f548099e480497a7a1f4ca4725c83db300b8d",
   ]
 }
 

k3s_cluster/k3s.tf

@@ -33,7 +33,7 @@ locals {
 
 }
 
-resource null_resource install_manifests {
+resource "null_resource" "install_manifests" {
   triggers = {
     server_id = hcloud_server.server.id
 
@@ -53,29 +53,29 @@ resource null_resource install_manifests {
   }
 
 
-  provisioner remote-exec {
+  provisioner "remote-exec" {
     inline = [local.manifest_hcloud_csi_driver]
   }
 
-  provisioner remote-exec {
+  provisioner "remote-exec" {
     inline = [local.manifest_cert_manager_crds]
   }
 
-  provisioner remote-exec {
+  provisioner "remote-exec" {
     inline = [local.manifest_helm_operator_crds]
   }
 
-  provisioner file {
+  provisioner "file" {
     content     = local.manifest_cert_manager
     destination = "${local.k3s_manifest_folder}/cert-manager.yaml"
   }
 
-  provisioner file {
+  provisioner "file" {
     content     = local.manifest_flux
     destination = "${local.k3s_manifest_folder}/flux.yaml"
   }
 
-  provisioner file {
+  provisioner "file" {
     content     = local.manifest_helm_operator
     destination = "${local.k3s_manifest_folder}/helm-operator.yaml"
   }

k3s_cluster_v2/agents.tf

@@ -0,0 +1,22 @@
+resource "hcloud_server" "agents" {
+  count = var.compute_count
+  name  = "k3s-agent-${count.index}"
+
+  image       = data.hcloud_image.ubuntu.name
+  server_type = var.compute_server_type
+  location    = var.server_location
+
+  ssh_keys = [data.hcloud_ssh_key.default.id]
+  labels = {
+    provisioner = "terraform",
+    engine      = "k3s",
+    node_type   = "agent",
+  }
+}
+
+resource "hcloud_server_network" "agents_network" {
+  count     = length(hcloud_server.agents)
+  server_id = hcloud_server.agents[count.index].id
+  subnet_id = hcloud_network_subnet.k3s_nodes.id
+  ip        = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + var.control_count + count.index)
+}

k3s_cluster_v2/k3s.tf

@@ -0,0 +1,123 @@
+module "k3s" {
+  source = "xunleii/k3s/module"
+
+  depends_on_    = hcloud_server.agents
+  k3s_version    = var.install_k3s_version
+  cluster_domain = "cluster.local"
+  cidr = {
+    pods     = "10.42.0.0/16"
+    services = "10.43.0.0/16"
+  }
+  drain_timeout  = "30s"
+  managed_fields = ["label", "taint"] // ignore annotations
+
+  global_flags = [
+    "--kubelet-arg cloud-provider=external" // required to use https://github.com/hetznercloud/hcloud-cloud-controller-manager
+  ]
+
+  servers = {
+    for i in range(length(hcloud_server.control_planes)) :
+    hcloud_server.control_planes[i].name => {
+      ip = hcloud_server_network.control_planes[i].ip
+      connection = {
+        type = "ssh"
+        host = hcloud_server.control_planes[i].ipv4_address
+
+        agent = true
+      }
+      flags       = ["--disable-cloud-controller", "--tls-san ${var.domain}"]
+      annotations = { "server_id" : i } // theses annotations will not be managed by this module
+    }
+  }
+
+  agents = {
+    for i in range(length(hcloud_server.agents)) :
+    "${hcloud_server.agents[i].name}_node" => {
+      name = hcloud_server.agents[i].name
+      ip   = hcloud_server_network.agents_network[i].ip
+      connection = {
+        type = "ssh"
+        host = hcloud_server.agents[i].ipv4_address
+      }
+
+      labels = {}
+      taints = {}
+    }
+  }
+}
+
+provider "kubernetes" {
+  host                   = module.k3s.kubernetes.api_endpoint
+  cluster_ca_certificate = module.k3s.kubernetes.cluster_ca_certificate
+  client_certificate     = module.k3s.kubernetes.client_certificate
+  client_key             = module.k3s.kubernetes.client_key
+}
+
+resource "kubernetes_service_account" "bootstrap" {
+  depends_on = [module.k3s.kubernetes_ready]
+
+  metadata {
+    name      = "bootstrap"
+    namespace = "default"
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "boostrap" {
+  depends_on = [module.k3s.kubernetes_ready]
+
+  metadata {
+    name = "bootstrap"
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = "bootstrap"
+    namespace = "default"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "admin"
+  }
+}
+
+data "kubernetes_secret" "sa_credentials" {
+  metadata {
+    name      = kubernetes_service_account.bootstrap.default_secret_name
+    namespace = "default"
+  }
+}
+
+## hcloud-cloud-controller-manager is necessary for cluster bootstrap
+
+data "http" "hcloud_cloud_controller_manager" {
+  url = "https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/v1.12.1/deploy/ccm-networks.yaml"
+}
+
+locals {
+  hccm_all_manifests = split("---", data.http.hcloud_cloud_controller_manager.body)
+
+  // first element is only comment
+  hccm_actual_manifests = slice(local.hccm_all_manifests, 1, length(local.hccm_all_manifests))
+}
+
+resource "kubernetes_manifest" "hcloud_cloud_controller_manager" {
+  for_each = toset(
+    local.hccm_actual_manifests
+  )
+
+  manifest = yamldecode(each.key)
+}
+
+resource "kubernetes_secret" "hcloud_token" {
+  metadata {
+    name      = "hcloud"
+    namespace = "kube-system"
+  }
+
+  data = {
+    token   = var.hcloud_ccm_token
+    network = hcloud_network.k3s.id
+  }
+}

k3s_cluster_v2/main.tf

@@ -0,0 +1,46 @@
+data "hcloud_ssh_key" "default" {
+  name = "default"
+}
+
+resource "hcloud_network" "k3s" {
+  name     = "k3s-network"
+  ip_range = "10.0.0.0/8"
+}
+
+resource "hcloud_network_subnet" "k3s_nodes" {
+  type         = "cloud"
+  network_id   = hcloud_network.k3s.id
+  network_zone = "eu-central"
+  ip_range     = "10.254.1.0/24"
+}
+
+resource "hcloud_network_subnet" "lb" {
+  type         = "cloud"
+  network_id   = hcloud_network.k3s.id
+  network_zone = "eu-central"
+  ip_range     = "10.254.2.0/24"
+}
+
+data "hcloud_image" "ubuntu" {
+  name = var.server_image
+}
+
+### Loadbalancer
+
+resource "hcloud_load_balancer" "k3s" {
+  name               = "k3s"
+  load_balancer_type = var.load_balancer_type
+  location           = var.server_location
+}
+
+resource "hcloud_load_balancer_network" "k3s" {
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  subnet_id        = hcloud_network_subnet.lb.id
+}
+
+
+resource "hcloud_rdns" "k3s" {
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  ip_address       = hcloud_load_balancer.k3s.ipv4
+  dns_ptr          = var.domain
+}

k3s_cluster_v2/output.tf

@@ -0,0 +1,9 @@
+output "summary" {
+  value = module.k3s.summary
+}
+
+output "bootstrap_sa" {
+  description = "Bootstrap ServiceAccount. Can be used by Terraform to provision this cluster."
+  value       = data.kubernetes_secret.sa_credentials.data
+  sensitive   = true
+}

k3s_cluster_v2/server.tf

@@ -0,0 +1,38 @@
+resource "hcloud_server" "control_planes" {
+  count = var.control_count
+  name  = "k3s-control-plane-${count.index}"
+
+  image       = data.hcloud_image.ubuntu.name
+  server_type = var.control_server_type
+  location    = var.server_location
+
+  ssh_keys = [data.hcloud_ssh_key.default.id]
+  labels = {
+    provisioner = "terraform",
+    engine      = "k3s",
+    node_type   = "control-plane"
+  }
+}
+
+resource "hcloud_server_network" "control_planes" {
+  count     = var.control_count
+  subnet_id = hcloud_network_subnet.k3s_nodes.id
+  server_id = hcloud_server.control_planes[count.index].id
+  ip        = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + count.index)
+}
+
+# LB
+
+resource "hcloud_load_balancer_service" "api" {
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  protocol         = "tcp"
+  listen_port      = 6443
+  destination_port = 6443
+}
+
+resource "hcloud_load_balancer_target" "api" {
+  count            = var.control_count
+  type             = "server"
+  load_balancer_id = hcloud_load_balancer.k3s.id
+  server_id        = hcloud_server.control_planes[count.index].id
+}

k3s_cluster_v2/variables.tf

@@ -0,0 +1,57 @@
+variable "name" {
+  type = string
+}
+
+variable "server_image" {
+  type = string
+  # With ubuntu-20.04 k3s crashes on start (v1.17.4+k3s1)
+  default = "ubuntu-18.04"
+}
+
+variable "server_location" {
+  type = string
+}
+
+variable "control_server_type" {
+  type    = string
+  default = "cx21"
+}
+
+variable "compute_server_type" {
+  type    = string
+  default = "cpx21"
+}
+
+variable "control_count" {
+  description = "Number of control plane nodes."
+  default     = 3
+}
+
+variable "compute_count" {
+  type    = number
+  default = 1
+}
+
+variable "load_balancer_type" {
+  type    = string
+  default = "lb11"
+}
+
+variable "domain" {
+  type = string
+}
+
+variable "install_k3s_version" {
+  type    = string
+  default = "v1.22.4+k3s1"
+}
+
+variable "ssh_key" {
+  description = "SSH public Key content needed to provision the instances."
+  type        = string
+}
+
+variable "hcloud_ccm_token" {
+  description = "HCloud API Token used by the hcloud-cloud-controller-manager"
+  type        = string
+}

k3s_cluster_v2/versions.tf

@@ -0,0 +1,22 @@
+
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    hcloud = {
+      source = "hetznercloud/hcloud"
+    }
+    tls = {
+      source = "hashicorp/tls"
+    }
+    template = {
+      source = "hashicorp/template"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+    null = {
+      source = "hashicorp/null"
+    }
+  }
+}

main.tf

@@ -2,7 +2,7 @@ locals {
   cluster_name = "home-cloud"
 }
 
-module k3s_cluster {
+module "k3s_cluster" {
   source = "./k3s_cluster"
 
   name                    = local.cluster_name
@@ -21,3 +21,24 @@ module k3s_cluster {
     hcloud = hcloud
   }
 }
+
+module "k3s_cluster_v2" {
+  source = "./k3s_cluster_v2"
+
+  name                = local.cluster_name
+  server_image        = "ubuntu-20.04"
+  server_location     = "nbg1"
+  control_server_type = "cx21"
+  compute_server_type = "cpx31"
+  load_balancer_type  = "lb11"
+  install_k3s_version = "v1.22.4+k3s1"
+  control_count       = 1
+  compute_count       = 1
+  domain              = "c2.apricote.de"
+  ssh_key             = file("~/.ssh/id_rsa.pub")
+  hcloud_ccm_token    = var.hcloud_ccm_token
+
+  providers = {
+    hcloud = hcloud
+  }
+}

variables.tf

@@ -1,3 +1,7 @@
-variable hcloud_csi_driver_token {
+variable "hcloud_csi_driver_token" {
+  type = string
+}
+
+variable "hcloud_ccm_token" {
   type = string
 }