diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 1345707..93cf2fc 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -1,39 +1,79 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/null" { - version = "3.1.0" +provider "registry.terraform.io/hashicorp/http" { + version = "1.2.0" + constraints = "~> 1.2" hashes = [ - "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=", - "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", - "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", - "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", - "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", - "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", - "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", - "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", - "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", - "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", - "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", - "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", + "h1:g5/qN61yBHRqpGVm08BnBp1O3vVf1eIpMglYvsY13XE=", + "zh:12df09a69dbf8219d9544466928abc65cc8cf38a26258a7586921e1e791a89a2", + "zh:3a1a57be455011ae09df92939a2e4ea1ac1d5fd74b4617889b10a03e83b0238d", + "zh:48f380c5e9c3abb69b7ecc40ed68f24c9ad42727d58ad530cc61f2d0524d2664", + "zh:4d3102f731915609dd5dda55d1525c3ac3be20fae37feba2f0e2770c6e71c5a3", + "zh:5b8c1bbe08b9f0e5f9c0d498c4f0316807595a6d55b8c194fa36d692e66f8afe", + "zh:8d3a13bbf852a884905dce915665d8ee817fd436d088226ed4fabbc6db57d0fe", + "zh:9025dfd565785c9424243bf78546c9c1ff3064b732ad178d9142831f5c5bea66", + "zh:a370ac52da85934af38b60858be3e91f44a6a454b007f607b1b8a0ea307e2d27", + "zh:ab9a1be3a3c0c760d33d0a7234909f92282f704c8145443089abbf0fddda2d85", + "zh:d97ca45677d77eb49864887077e8457f518bd789f0609a19796b2fce70d1aadd", + "zh:ec2f2762eeabbd2706b26f1cb03aea5301f32468147d81cc84c823ef927bc292", + "zh:ef6851bde213957d5331039d9dc88c89d2d3182793387dae15419343c853664a", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.7.0" + hashes = [ + "h1:Cd7Kh8JmqhevGx+OsZ33QSa5hBHOs7CT9sfqTR5NEGM=", + "zh:0ee4ed19d951cf94a1b27bf813827bb5c51b48ac53efe827e77aa077e9d66021", + "zh:134ae78a19a43f94c2958e87ee8c6c1ae4593c145569bf71bf58a357882c18af", + "zh:191345c9c0808499378d6ac33237f41d4509769b06367630acbf1586e0fe1540", + "zh:1c058459ca1705dd97646d06c02df648a19a12789024a16155209bbdd75d03d6", + "zh:33ac31a72649b85962cf3404edd08e917704b225eb4064d63cfee5704635da56", + "zh:3dcc51d76711ff58d77a42811d8f65834fdb3632d44f56f2fe58eaca07c83a8a", + "zh:5d5ea922c72d2eb4abc67d9a34fcf92b3af33049325bb7384e772a9db90120dd", + "zh:9b02cda7aad45408fa2acbc844505d6a72c3998eb28e3568c1d35e142acd1ffd", + "zh:b7f4652ec526bc3e7a846a40b136dc4d1d873f643329056eeb8b7ac5dfb4e11e", + "zh:c8d1617426ca3d91bc0c5188f353dbca55ca777c28830e9bb0b79baef227885e", + "zh:ed7064a03e15ca6b2f026064f17fa3956775e34a930e5265399ffa7f6ef22ded", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "2.1.2" + constraints = "~> 2.1" + hashes = [ + "h1:CFnENdqQu4g3LJNevA32aDxcUz2qGkRGQpFfkI8TCdE=", + "zh:0cc7236c1fbf971b8bad1540a7d0c5ac4579248239fd1034c023b0b660a8d1d5", + "zh:16fc2d9b10cf9e5123bf956e7032c338cc93a03be1ca2e9f3d3b7014c0e866c7", + "zh:1e26465ff40ded59cef1a9e745752eef9744471e69094d12f8bc40a060e8cdb9", + "zh:3c7afd28076245f455d4348af6c124b73409722be4a73680d4e4709a4f25ea91", + "zh:4190a92567efaa444527f19b28d65fac1a01aeba907013b5dceacd9ba2a23709", + "zh:677963437316b088fc1aac70fe7d608d2917c46530c4a25ec86a43e9acaf2811", + "zh:69fe15f6b851ff82700bc749c50a9299770515617e677c18cba2cb697daaff36", + "zh:6b505cc60cc1494e1cab61590bca127b06dd894d0b2a7dcacd23862bce1f492b", + "zh:719aa11ad7be974085af595089478eb24d5a021bc7b08364fa6745d9eb473dac", + "zh:7375b02189e14efbfaab994cd05d81e3ff8e46041fae778598b3903114093a3e", + "zh:c406573b2084a08f8faa0451923fbeb1ca6c5a5598bf039589ec2db13aacc622", + "zh:fb11299a3b20711595713d126abbbe78c554eb5995b02db536e9253686798fb6", ] } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.0" + version = "2.3.1" + constraints = "~> 2.2" hashes = [ - "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", - "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", - "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", - "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", - "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", - "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", - "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", - "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", - "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", - "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", - "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", - "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + "h1:bPBDLMpQzOjKhDlP9uH2UPIz9tSjcbCtLdiJ5ASmCx4=", + "zh:322ec2b56765162c193d0ff0f7634351bf4ca09d719558df82b92d31059995a1", + "zh:34c500857d14148bc6f7f21f9fc0d832ba31d00e1b911b0437d99dda29aeb108", + "zh:a3372514add474fcb07d2026a49b0c0d8a699e365f540e43e5f590eb924b95f7", + "zh:a54acd50f13f38a86454acad16b3487701608188fd2cc454fd48beee00127ae5", + "zh:b1285c47ab01fc11b6086f080da7d17b9155b3fac6a4175948a2abb5d52c60af", + "zh:c7e7bd46218cf504ca31d8a1e4eab5cad625c9951fb89d4f56861d5bddec9afb", + "zh:d9f95764480d7b884db7247a5ee9d1c72aaf1c1cde3d7b5e34bc33c4ef5ccf48", + "zh:ddf11fb807d61ff93c08206733129bf9668fc0c4adedf8a6ac38a7c382a24b35", + "zh:f631a0c1ffa94991d87cd5fe4e08c0dd6d36c780585f35a582c05ee2affb7e86", + "zh:fb5f5fb19e8a9ff73dbdee85a97f548099e480497a7a1f4ca4725c83db300b8d", ] } diff --git a/k3s_cluster/k3s.tf b/k3s_cluster/k3s.tf index 35e4815..16ace0e 100644 --- a/k3s_cluster/k3s.tf +++ b/k3s_cluster/k3s.tf @@ -33,7 +33,7 @@ locals { } -resource null_resource install_manifests { +resource "null_resource" "install_manifests" { triggers = { server_id = hcloud_server.server.id @@ -53,29 +53,29 @@ resource null_resource install_manifests { } - provisioner remote-exec { + provisioner "remote-exec" { inline = [local.manifest_hcloud_csi_driver] } - provisioner remote-exec { + provisioner "remote-exec" { inline = [local.manifest_cert_manager_crds] } - provisioner remote-exec { + provisioner "remote-exec" { inline = [local.manifest_helm_operator_crds] } - provisioner file { + provisioner "file" { content = local.manifest_cert_manager destination = "${local.k3s_manifest_folder}/cert-manager.yaml" } - provisioner file { + provisioner "file" { content = local.manifest_flux destination = "${local.k3s_manifest_folder}/flux.yaml" } - provisioner file { + provisioner "file" { content = local.manifest_helm_operator destination = "${local.k3s_manifest_folder}/helm-operator.yaml" } diff --git a/k3s_cluster_v2/agents.tf b/k3s_cluster_v2/agents.tf new file mode 100644 index 0000000..3454774 --- /dev/null +++ b/k3s_cluster_v2/agents.tf @@ -0,0 +1,22 @@ +resource "hcloud_server" "agents" { + count = var.compute_count + name = "k3s-agent-${count.index}" + + image = data.hcloud_image.ubuntu.name + server_type = var.compute_server_type + location = var.server_location + + ssh_keys = [data.hcloud_ssh_key.default.id] + labels = { + provisioner = "terraform", + engine = "k3s", + node_type = "agent", + } +} + +resource "hcloud_server_network" "agents_network" { + count = length(hcloud_server.agents) + server_id = hcloud_server.agents[count.index].id + subnet_id = hcloud_network_subnet.k3s_nodes.id + ip = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + var.control_count + count.index) +} diff --git a/k3s_cluster_v2/k3s.tf b/k3s_cluster_v2/k3s.tf new file mode 100644 index 0000000..926371f --- /dev/null +++ b/k3s_cluster_v2/k3s.tf @@ -0,0 +1,123 @@ +module "k3s" { + source = "xunleii/k3s/module" + + depends_on_ = hcloud_server.agents + k3s_version = var.install_k3s_version + cluster_domain = "cluster.local" + cidr = { + pods = "10.42.0.0/16" + services = "10.43.0.0/16" + } + drain_timeout = "30s" + managed_fields = ["label", "taint"] // ignore annotations + + global_flags = [ + "--kubelet-arg cloud-provider=external" // required to use https://github.com/hetznercloud/hcloud-cloud-controller-manager + ] + + servers = { + for i in range(length(hcloud_server.control_planes)) : + hcloud_server.control_planes[i].name => { + ip = hcloud_server_network.control_planes[i].ip + connection = { + type = "ssh" + host = hcloud_server.control_planes[i].ipv4_address + + agent = true + } + flags = ["--disable-cloud-controller", "--tls-san ${var.domain}"] + annotations = { "server_id" : i } // theses annotations will not be managed by this module + } + } + + agents = { + for i in range(length(hcloud_server.agents)) : + "${hcloud_server.agents[i].name}_node" => { + name = hcloud_server.agents[i].name + ip = hcloud_server_network.agents_network[i].ip + connection = { + type = "ssh" + host = hcloud_server.agents[i].ipv4_address + } + + labels = {} + taints = {} + } + } +} + +provider "kubernetes" { + host = module.k3s.kubernetes.api_endpoint + cluster_ca_certificate = module.k3s.kubernetes.cluster_ca_certificate + client_certificate = module.k3s.kubernetes.client_certificate + client_key = module.k3s.kubernetes.client_key +} + +resource "kubernetes_service_account" "bootstrap" { + depends_on = [module.k3s.kubernetes_ready] + + metadata { + name = "bootstrap" + namespace = "default" + } +} + +resource "kubernetes_cluster_role_binding" "boostrap" { + depends_on = [module.k3s.kubernetes_ready] + + metadata { + name = "bootstrap" + } + + subject { + kind = "ServiceAccount" + name = "bootstrap" + namespace = "default" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "admin" + } +} + +data "kubernetes_secret" "sa_credentials" { + metadata { + name = kubernetes_service_account.bootstrap.default_secret_name + namespace = "default" + } +} + +## hcloud-cloud-controller-manager is necessary for cluster bootstrap + +data "http" "hcloud_cloud_controller_manager" { + url = "https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/v1.12.1/deploy/ccm-networks.yaml" +} + +locals { + hccm_all_manifests = split("---", data.http.hcloud_cloud_controller_manager.body) + + // first element is only comment + hccm_actual_manifests = slice(local.hccm_all_manifests, 1, length(local.hccm_all_manifests)) +} + +resource "kubernetes_manifest" "hcloud_cloud_controller_manager" { + for_each = toset( + local.hccm_actual_manifests + ) + + manifest = yamldecode(each.key) +} + +resource "kubernetes_secret" "hcloud_token" { + metadata { + name = "hcloud" + namespace = "kube-system" + } + + data = { + token = var.hcloud_ccm_token + network = hcloud_network.k3s.id + } +} diff --git a/k3s_cluster_v2/main.tf b/k3s_cluster_v2/main.tf new file mode 100755 index 0000000..86579ef --- /dev/null +++ b/k3s_cluster_v2/main.tf @@ -0,0 +1,46 @@ +data "hcloud_ssh_key" "default" { + name = "default" +} + +resource "hcloud_network" "k3s" { + name = "k3s-network" + ip_range = "10.0.0.0/8" +} + +resource "hcloud_network_subnet" "k3s_nodes" { + type = "cloud" + network_id = hcloud_network.k3s.id + network_zone = "eu-central" + ip_range = "10.254.1.0/24" +} + +resource "hcloud_network_subnet" "lb" { + type = "cloud" + network_id = hcloud_network.k3s.id + network_zone = "eu-central" + ip_range = "10.254.2.0/24" +} + +data "hcloud_image" "ubuntu" { + name = var.server_image +} + +### Loadbalancer + +resource "hcloud_load_balancer" "k3s" { + name = "k3s" + load_balancer_type = var.load_balancer_type + location = var.server_location +} + +resource "hcloud_load_balancer_network" "k3s" { + load_balancer_id = hcloud_load_balancer.k3s.id + subnet_id = hcloud_network_subnet.lb.id +} + + +resource "hcloud_rdns" "k3s" { + load_balancer_id = hcloud_load_balancer.k3s.id + ip_address = hcloud_load_balancer.k3s.ipv4 + dns_ptr = var.domain +} diff --git a/k3s_cluster_v2/output.tf b/k3s_cluster_v2/output.tf new file mode 100644 index 0000000..6a2e5ce --- /dev/null +++ b/k3s_cluster_v2/output.tf @@ -0,0 +1,9 @@ +output "summary" { + value = module.k3s.summary +} + +output "bootstrap_sa" { + description = "Bootstrap ServiceAccount. Can be used by Terraform to provision this cluster." + value = data.kubernetes_secret.sa_credentials.data + sensitive = true +} diff --git a/k3s_cluster_v2/server.tf b/k3s_cluster_v2/server.tf new file mode 100644 index 0000000..9c94ba2 --- /dev/null +++ b/k3s_cluster_v2/server.tf @@ -0,0 +1,38 @@ +resource "hcloud_server" "control_planes" { + count = var.control_count + name = "k3s-control-plane-${count.index}" + + image = data.hcloud_image.ubuntu.name + server_type = var.control_server_type + location = var.server_location + + ssh_keys = [data.hcloud_ssh_key.default.id] + labels = { + provisioner = "terraform", + engine = "k3s", + node_type = "control-plane" + } +} + +resource "hcloud_server_network" "control_planes" { + count = var.control_count + subnet_id = hcloud_network_subnet.k3s_nodes.id + server_id = hcloud_server.control_planes[count.index].id + ip = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + count.index) +} + +# LB + +resource "hcloud_load_balancer_service" "api" { + load_balancer_id = hcloud_load_balancer.k3s.id + protocol = "tcp" + listen_port = 6443 + destination_port = 6443 +} + +resource "hcloud_load_balancer_target" "api" { + count = var.control_count + type = "server" + load_balancer_id = hcloud_load_balancer.k3s.id + server_id = hcloud_server.control_planes[count.index].id +} diff --git a/k3s_cluster_v2/variables.tf b/k3s_cluster_v2/variables.tf new file mode 100644 index 0000000..a0a8813 --- /dev/null +++ b/k3s_cluster_v2/variables.tf @@ -0,0 +1,57 @@ +variable "name" { + type = string +} + +variable "server_image" { + type = string + # With ubuntu-20.04 k3s crashes on start (v1.17.4+k3s1) + default = "ubuntu-18.04" +} + +variable "server_location" { + type = string +} + +variable "control_server_type" { + type = string + default = "cx21" +} + +variable "compute_server_type" { + type = string + default = "cpx21" +} + +variable "control_count" { + description = "Number of control plane nodes." + default = 3 +} + +variable "compute_count" { + type = number + default = 1 +} + +variable "load_balancer_type" { + type = string + default = "lb11" +} + +variable "domain" { + type = string +} + +variable "install_k3s_version" { + type = string + default = "v1.22.4+k3s1" +} + +variable "ssh_key" { + description = "SSH public Key content needed to provision the instances." + type = string +} + +variable "hcloud_ccm_token" { + description = "HCloud API Token used by the hcloud-cloud-controller-manager" + type = string +} diff --git a/k3s_cluster_v2/versions.tf b/k3s_cluster_v2/versions.tf new file mode 100644 index 0000000..b8e33b6 --- /dev/null +++ b/k3s_cluster_v2/versions.tf @@ -0,0 +1,22 @@ + +terraform { + required_version = ">= 1.0" + + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + } + tls = { + source = "hashicorp/tls" + } + template = { + source = "hashicorp/template" + } + random = { + source = "hashicorp/random" + } + null = { + source = "hashicorp/null" + } + } +} diff --git a/main.tf b/main.tf index a958129..530c4b5 100644 --- a/main.tf +++ b/main.tf @@ -2,7 +2,7 @@ locals { cluster_name = "home-cloud" } -module k3s_cluster { +module "k3s_cluster" { source = "./k3s_cluster" name = local.cluster_name @@ -21,3 +21,24 @@ module k3s_cluster { hcloud = hcloud } } + +module "k3s_cluster_v2" { + source = "./k3s_cluster_v2" + + name = local.cluster_name + server_image = "ubuntu-20.04" + server_location = "nbg1" + control_server_type = "cx21" + compute_server_type = "cpx31" + load_balancer_type = "lb11" + install_k3s_version = "v1.22.4+k3s1" + control_count = 1 + compute_count = 1 + domain = "c2.apricote.de" + ssh_key = file("~/.ssh/id_rsa.pub") + hcloud_ccm_token = var.hcloud_ccm_token + + providers = { + hcloud = hcloud + } +} diff --git a/variables.tf b/variables.tf index c958740..6790ffb 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,7 @@ -variable hcloud_csi_driver_token { +variable "hcloud_csi_driver_token" { + type = string +} + +variable "hcloud_ccm_token" { type = string }