diff --git a/.gitignore b/.gitignore index 23fcdf8..b421c6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ keys/id_terraform* credentials.tfvars terraform.tfstate* -.terraform \ No newline at end of file +.terraform + +kubeconfig.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index 9b8eed6..18d380a 100644 --- a/Makefile +++ b/Makefile @@ -17,9 +17,15 @@ lint: init $(VALIDATE) services/bitwarden $(VALIDATE) . -init: keys +init: keys/id_terraform $(TF) init -keys: keys/id_terraform +keys/id_terraform: echo "No private key found! Generating Terraform SSH Keys." - ./bootstrap-keys.sh \ No newline at end of file + ./scripts/bootstrap-keys.sh + +kubeconfig: keys/id_terraform + + scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i keys/id_terraform root@`terraform output cluster_public_ip`:/etc/rancher/k3s/k3s.yaml ./kubeconfig.yaml + sed -i "s/127.0.0.1/`terraform output cluster_public_ip`/g" ./kubeconfig.yaml + sed -i "s/default/`terraform output cluster_name`/g" ./kubeconfig.yaml \ No newline at end of file diff --git a/bootstrap-keys.sh b/bootstrap-keys.sh deleted file mode 100755 index 8ad8366..0000000 --- a/bootstrap-keys.sh +++ /dev/null @@ -1,2 +0,0 @@ -ssh-keygen -t rsa -C "terraform@narando.de" -f keys/id_terraform -chmod 600 keys/id_terraform* \ No newline at end of file diff --git a/dashboard.tf b/dashboard.tf deleted file mode 100644 index 25f7d18..0000000 --- a/dashboard.tf +++ /dev/null @@ -1,32 +0,0 @@ -resource "kubernetes_service_account" "dashboard" { - metadata { - name = "dashboard-admin" - namespace = "kube-system" - - labels = { - app = "dashboard" - } - } -} - -resource "kubernetes_cluster_role_binding" "dashboard" { - metadata { - name = "dashboard-admin" - - labels = { - app = "dashboard" - } - } - - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "cluster-admin" - } - - subject { - kind = "ServiceAccount" - name = "dashboard-admin" - namespace = "kube-system" - } -} diff --git a/k3s_cluster/cloud_init.tf b/k3s_cluster/cloud_init.tf new file mode 100644 index 0000000..c67d56a --- /dev/null +++ b/k3s_cluster/cloud_init.tf @@ -0,0 +1,66 @@ +locals { + install_k3s_version = var.install_k3s_version + k3s_storage_endpoint = "sqlite" + k3s_cluster_secret = var.k3s_cluster_secret != null ? var.k3s_cluster_secret : random_password.k3s_cluster_secret.result + k3s_tls_san = "--tls-san ${var.domain}" + floating_ip_use_netdata = var.server_image == "ubuntu-20.04" +} + +resource "random_password" "k3s_cluster_secret" { + length = 30 + special = false +} + +data "template_cloudinit_config" "k3s_server" { + gzip = false + base64_encode = false + + # Main cloud-config configuration file. + part { + filename = "init.cfg" + content_type = "text/cloud-config" + content = templatefile("${path.module}/files/cloud-config-base.yaml", {}) + } + + part { + content_type = "text/x-shellscript" + content = templatefile("${path.module}/files/setup-floating-ip.sh", { + floating_ip = hcloud_floating_ip.server.ip_address, + use_netdata = local.floating_ip_use_netdata + }) + } + + part { + content_type = "text/x-shellscript" + content = templatefile("${path.module}/files/k3s-install.sh", { + is_k3s_server = true, + install_k3s_version = local.install_k3s_version, + k3s_cluster_secret = local.k3s_cluster_secret, + k3s_url = hcloud_floating_ip.server.ip_address, + k3s_tls_san = local.k3s_tls_san, + }) + } +} + +data "template_cloudinit_config" "k3s_agent" { + gzip = false + base64_encode = false + + # Main cloud-config configuration file. + part { + filename = "init.cfg" + content_type = "text/cloud-config" + content = templatefile("${path.module}/files/cloud-config-base.yaml", {}) + } + + part { + content_type = "text/x-shellscript" + content = templatefile("${path.module}/files/k3s-install.sh", { + is_k3s_server = false, + install_k3s_version = local.install_k3s_version, + k3s_cluster_secret = local.k3s_cluster_secret, + k3s_url = hcloud_floating_ip.server.ip_address, + k3s_tls_san = local.k3s_tls_san, + }) + } +} diff --git a/k3s_cluster/files/cloud-config-base.yaml b/k3s_cluster/files/cloud-config-base.yaml new file mode 100644 index 0000000..bda5397 --- /dev/null +++ b/k3s_cluster/files/cloud-config-base.yaml @@ -0,0 +1,5 @@ +#cloud-config +runcmd: + - apt-get update + - apt-get install -y software-properties-common + - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y diff --git a/k3s_cluster/files/k3s-install.sh b/k3s_cluster/files/k3s-install.sh new file mode 100644 index 0000000..873c64c --- /dev/null +++ b/k3s_cluster/files/k3s-install.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +until ( \ + curl -sfL https://get.k3s.io | \ + INSTALL_K3S_VERSION='v${install_k3s_version}' \ + K3S_CLUSTER_SECRET='${k3s_cluster_secret}' \ + INSTALL_K3S_EXEC='%{ if is_k3s_server } ${k3s_tls_san} %{ endif }' \ + %{ if !is_k3s_server } K3S_URL='https://${k3s_url}:6443'%{ endif } \ + sh - \ + ); do + echo 'k3s did not install correctly' + sleep 2 +done + +%{ if is_k3s_server } +until kubectl get pods -A | grep 'Running'; +do + echo 'Waiting for k3s startup' + sleep 5 +done +%{ endif } \ No newline at end of file diff --git a/k3s_cluster/files/k8s-apps/cert-manager-crds.sh b/k3s_cluster/files/k8s-apps/cert-manager-crds.sh new file mode 100644 index 0000000..63c7e4e --- /dev/null +++ b/k3s_cluster/files/k8s-apps/cert-manager-crds.sh @@ -0,0 +1,5 @@ +#!/bin/bash +MANIFEST_FILE=https://github.com/jetstack/cert-manager/releases/download/${version}/cert-manager.crds.yaml +K3S_MANIFEST_FOLDER=${k3s_manifest_folder} + +curl -sfL $MANIFEST_FILE > $K3S_MANIFEST_FOLDER/cert-manager-crds.yml diff --git a/k3s_cluster/files/k8s-apps/cert-manager.yaml b/k3s_cluster/files/k8s-apps/cert-manager.yaml new file mode 100644 index 0000000..491587b --- /dev/null +++ b/k3s_cluster/files/k8s-apps/cert-manager.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager + +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cert-manager + namespace: kube-system +spec: + chart: cert-manager + repo: https://charts.jetstack.io + version: ${version} + targetNamespace: cert-manager + set: + ingressShim.defaultIssuerName: "letsencrypt-prod" + ingressShim.defaultIssuerKind: "ClusterIssuer" + +--- +apiVersion: cert-manager.io/v1alpha2 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + # You must replace this email address with your own. + # Let's Encrypt will use this to contact you about expiring + # certificates, and issues related to your account. + email: ${email} + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource that will be used to store the account's private key. + name: letsencrypt-prod-cluster-issuer-account + # Add a single challenge solver, HTTP01 using nginx + solvers: + - http01: + ingress: {} diff --git a/k3s_cluster/files/k8s-apps/hcloud-csi-driver.sh b/k3s_cluster/files/k8s-apps/hcloud-csi-driver.sh new file mode 100644 index 0000000..689150b --- /dev/null +++ b/k3s_cluster/files/k8s-apps/hcloud-csi-driver.sh @@ -0,0 +1,16 @@ +#!/bin/bash +MANIFEST_FILE=https://raw.githubusercontent.com/hetznercloud/csi-driver/${version}/deploy/kubernetes/hcloud-csi.yml +K3S_MANIFEST_FOLDER=${k3s_manifest_folder} + + +curl -sfL $MANIFEST_FILE > $K3S_MANIFEST_FOLDER/hcloud-csi.yml + +cat < $K3S_MANIFEST_FOLDER/hcloud-csi-token.yml +apiVersion: v1 +kind: Secret +metadata: + name: hcloud-csi + namespace: kube-system +stringData: + token: ${token} +EOF \ No newline at end of file diff --git a/k3s_cluster/files/setup-floating-ip.sh b/k3s_cluster/files/setup-floating-ip.sh new file mode 100644 index 0000000..5eb523e --- /dev/null +++ b/k3s_cluster/files/setup-floating-ip.sh @@ -0,0 +1,20 @@ +#!/bin/bash +%{ if use_netdata } +cat >> /etc/netplan/60-floating.cfg <<- EOM +network: + version: 2 + ethernets: + eth0: + addresses: + - ${floating_ip}/32 +EOM +netplan apply +%{ else } +cat >> /etc/network/interfaces.d/99-floating.cfg <<- EOM +auto eth0:1 +iface eth0:1 inet static + address ${floating_ip} + netmask 255.255.255.255 +EOM +ifdown eth0:1 ; ifup eth0:1 +%{ endif } diff --git a/k3s_cluster/k3s.tf b/k3s_cluster/k3s.tf new file mode 100644 index 0000000..97ebe2e --- /dev/null +++ b/k3s_cluster/k3s.tf @@ -0,0 +1,51 @@ +locals { + k3s_manifest_folder = "/var/lib/rancher/k3s/server/manifests" + + manifest_hcloud_csi_driver = templatefile("${path.module}/files/k8s-apps/hcloud-csi-driver.sh", { + version = var.hcloud_csi_driver_version + token = var.hcloud_csi_driver_token + k3s_manifest_folder = local.k3s_manifest_folder + }) + + manifest_cert_manager_crds = templatefile("${path.module}/files/k8s-apps/cert-manager-crds.sh", { + version = var.cert_manager_version + k3s_manifest_folder = local.k3s_manifest_folder + }) + + manifest_cert_manager = templatefile("${path.module}/files/k8s-apps/cert-manager.yaml", { + version = var.cert_manager_version + email = var.letsencrypt_email + }) +} + +resource null_resource install_manifests { + triggers = { + server_id = hcloud_server.server.id + + # File hashes to trigger on update + hcloud_csi_driver = sha256(local.manifest_hcloud_csi_driver) + cert_manager_crds = sha256(local.manifest_cert_manager_crds) + cert_manager = sha256(local.manifest_cert_manager) + } + + connection { + type = "ssh" + host = hcloud_server.server.ipv4_address + private_key = var.ssh_keys[0] + } + + + provisioner remote-exec { + inline = [local.manifest_hcloud_csi_driver] + } + + provisioner remote-exec { + inline = [local.manifest_cert_manager_crds] + } + + provisioner file { + content = local.manifest_cert_manager + destination = "${local.k3s_manifest_folder}/cert-manager.yaml" + } +} + diff --git a/k3s_cluster/main.tf b/k3s_cluster/main.tf new file mode 100755 index 0000000..5a1a48a --- /dev/null +++ b/k3s_cluster/main.tf @@ -0,0 +1,93 @@ +data tls_public_key default { + count = length(var.ssh_keys) + + private_key_pem = var.ssh_keys[count.index] +} + +resource hcloud_ssh_key default { + count = length(var.ssh_keys) + + name = "${var.name}-${count.index}" + + public_key = data.tls_public_key.default[count.index].public_key_openssh +} + +resource hcloud_floating_ip server { + name = "${var.name}-server" + home_location = var.server_location + description = "Persistent IP for K3s Server. Used in Domain and for Ingress." + type = "ipv4" +} + +resource hcloud_rdns server { + floating_ip_id = hcloud_floating_ip.server.id + ip_address = hcloud_floating_ip.server.ip_address + dns_ptr = var.domain +} + +resource hcloud_floating_ip_assignment server { + floating_ip_id = hcloud_floating_ip.server.id + server_id = hcloud_server.server.id +} + +resource hcloud_server server { + name = "${var.name}-server" + image = var.server_image + server_type = var.control_server_type + location = var.server_location + + ssh_keys = hcloud_ssh_key.default.*.id + + connection { + type = "ssh" + host = self.ipv4_address + private_key = var.ssh_keys[0] + } + + user_data = data.template_cloudinit_config.k3s_server.rendered + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait", + ] + } +} + + +resource "random_id" "agent" { + count = var.compute_count + + keepers = { + image = var.server_image + server_type = var.control_server_type + user_data = sha256(data.template_cloudinit_config.k3s_agent.rendered) + } + + byte_length = 3 +} + +resource hcloud_server agent { + count = var.compute_count + + name = "${var.name}-agent-${random_id.agent[count.index].hex}" + image = random_id.agent[count.index].keepers.image + server_type = random_id.agent[count.index].keepers.server_type + location = var.server_location + + + ssh_keys = hcloud_ssh_key.default.*.id + + connection { + type = "ssh" + host = self.ipv4_address + private_key = var.ssh_keys[0] + } + + user_data = data.template_cloudinit_config.k3s_agent.rendered + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait", + ] + } +} diff --git a/k3s_cluster/output.tf b/k3s_cluster/output.tf new file mode 100644 index 0000000..1c4fb94 --- /dev/null +++ b/k3s_cluster/output.tf @@ -0,0 +1,3 @@ +output server_public_ip { + value = hcloud_floating_ip.server.ip_address +} diff --git a/k3s_cluster/variables.tf b/k3s_cluster/variables.tf new file mode 100644 index 0000000..f38c634 --- /dev/null +++ b/k3s_cluster/variables.tf @@ -0,0 +1,67 @@ +variable name { + type = string +} + +variable server_image { + type = string + # With ubuntu-20.04 k3s crashes on start (v1.17.4+k3s1) + default = "ubuntu-18.04" +} + +variable server_location { + type = string +} + +variable control_server_type { + type = string + default = "cx11" +} + +variable compute_server_type { + type = string + default = "cx21" +} + +variable compute_count { + type = number + default = 1 +} + +variable domain { + type = string +} + +variable letsencrypt_email { + type = string + default = "none@none.com" + description = "LetsEncrypt email address to use" +} + +variable ssh_keys { + type = list(string) + default = [] +} + +variable install_k3s_version { + type = string + default = "1.17.4+k3s1" +} + +variable k3s_cluster_secret { + type = string + default = null +} + +variable hcloud_csi_driver_version { + type = string + default = "v1.2.3" +} + +variable hcloud_csi_driver_token { + type = string +} + +variable cert_manager_version { + type = string + default = "v0.14.3" +} diff --git a/k3s_cluster/versions.tf b/k3s_cluster/versions.tf new file mode 100644 index 0000000..009e7a3 --- /dev/null +++ b/k3s_cluster/versions.tf @@ -0,0 +1,11 @@ + +terraform { + required_version = "~> 0.12.0" + + required_providers { + hcloud = "~> 1.2" + tls = "~> 2.1" + template = "~> 2.1" + random = "~> 2.2" + } +} diff --git a/main.tf b/main.tf old mode 100755 new mode 100644 index 7f8553d..8b718d2 --- a/main.tf +++ b/main.tf @@ -1,139 +1,18 @@ -resource hcloud_server control { - count = 3 - name = "control${count.index}" - image = "ubuntu-18.04" - server_type = "cx21" - - ssh_keys = ["${hcloud_ssh_key.terraform.id}"] - - connection { - private_key = "${file("./keys/id_terraform")}" - } - - user_data = <