feat: new k8s deployment (v3)

This commit is contained in:
Julian Tölle 2023-05-19 22:04:30 +02:00
parent 48c08f7262
commit b43d70812e
13 changed files with 64 additions and 532 deletions

99
.terraform.lock.hcl generated
View file

@ -1,84 +1,6 @@
# This file is maintained automatically by "terraform init". # This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates. # Manual edits may be lost in future updates.
provider "registry.terraform.io/fluxcd/flux" {
version = "0.23.0"
constraints = ">= 0.10.0"
hashes = [
"h1:y1GdLmpI2rWvSRnnrVL1KVMqMz6Y91qTVOyH6lCZDfw=",
"zh:01e40fae6b7a988a411034fce87f05070e471047d30f195a8cd82eb715bf84e1",
"zh:0234fc5d6fea07d50b00e208c4962ba6ad840f37ab7f8c885168fd7b146ef054",
"zh:16fcc7946e623652c5e46539ce75acc2f862977dcce74eb1395509368d548770",
"zh:2bc46e5edb225472f79997411ffeddc8078f951d104ae4affbd1254376f9f111",
"zh:2cf2a8c37ed8b47f67cced02636d9fec262450e4b6933998db00af01b04a703c",
"zh:2dedd76d22620db791907791984a9cf9aee70e6dff1544d11b60952809efc621",
"zh:4e2c701c1f44f8d5fce4f14549b89b55e2622048444cab3806f8784b204e0a9e",
"zh:646ce188b34da47c4ceecce9e8988ad38b4bfea9d840ec1020a13a109f760e02",
"zh:72589e390d85d874130b9824a7074b65d2df688a8d3610df33c52538e70601a5",
"zh:80af886f100f3f1f4c20da987c6ee55b2fdb0be2d643de4f27d44ac4a56b50e7",
"zh:ae9ccdab496588cac30c869aeaf8bb9520c988838df7169ce96ed97d70895f48",
"zh:beec8ec89268ed86bca72b8325e6065ef3d279c8bdbbc857f6f1e561f9c7069a",
"zh:cd9b3468276c7945daf4be9d49101b5dc207fb6dce899597dc41997ef89975c7",
"zh:dae9ee5b7d38694af609d0e13247f0c900d15e98da203f13a7855f6e11012e9c",
]
}
provider "registry.terraform.io/gavinbunney/kubectl" {
version = "1.14.0"
constraints = ">= 1.10.0"
hashes = [
"h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=",
"zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858",
"zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030",
"zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5",
"zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4",
"zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5",
"zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5",
"zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf",
"zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93",
"zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65",
]
}
provider "registry.terraform.io/hashicorp/kubernetes" {
version = "2.17.0"
hashes = [
"h1:I1L2R+OPgGSh+P6uBSycvvoyRIey/FqMwSvlJ9ccw0o=",
"zh:1cbafea8c404195d8ad2490d75dbeebef131563d3e38dec87231ceb3923a3012",
"zh:26d9584423ee77e607999b082de7d9dc3e937934aa83341e0832e7253caf4f51",
"zh:333527fc15fb43bbf1898a2f058598c596468a01d88c415627bb617878dc4d4d",
"zh:391b8c80e3115af485977d6e949d7260b7fc0b641089b884256bfd36a7077db2",
"zh:4d18ba55247486181759d60195777945bcd68e17ccd980820ca18e8a8b94aeb5",
"zh:607ae94d85d1c1ed3845bd71095daadea4b2468e16f57fa05c98eab0de6b14ae",
"zh:95c6cf22f8ef14e7a4f85e33cff5d6f11056c7880041b71d425d1b5ebbe246e7",
"zh:b077edcedb46a313b461ac1e49317872063b3871f2acbe1a50498612cefff387",
"zh:c6a7891683e44148b0c928fd4748b7abac727266ab551d679015f5fe8b72d1e6",
"zh:e5cebfdf873770c37a4304362003d3fea8d6c2fd819663ad121bc65bb81e4738",
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
"zh:feb19269e7c0de473ad412b37818b48da0cc91e5c93dd4c77a72676ca97a16b1",
]
}
provider "registry.terraform.io/hashicorp/null" {
version = "3.2.1"
constraints = "~> 3.0"
hashes = [
"h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
"zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
"zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
"zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
"zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
"zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
"zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
"zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
"zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
"zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
"zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
]
}
provider "registry.terraform.io/hashicorp/random" { provider "registry.terraform.io/hashicorp/random" {
version = "3.4.3" version = "3.4.3"
constraints = "~> 3.0" constraints = "~> 3.0"
@ -158,27 +80,6 @@ provider "registry.terraform.io/hetznercloud/hcloud" {
] ]
} }
provider "registry.terraform.io/integrations/github" {
version = "5.17.0"
hashes = [
"h1:CWw2DL8qmBp/LkqZAC3HiNFskw4bPyZYXgVgwUK7Lew=",
"zh:0caa38dab96d68621a1ae7087ca3b86f42aa0e6fc250f906299f1a34c9dd1e54",
"zh:1119f8dacb2da0de0735e9ae586702e5f9758b963e548b5fa09a9f216d00bbc4",
"zh:16bed2a93216aa573d1b2ff7cd371c9df3d454284204a4695d5b30f7325f49b3",
"zh:537d29a3a18d6b3a588c8878793d99d937d1e29466c02ce08536943a26931387",
"zh:664d83424cc8d12055806134e5d110b82f469fb5824d3c3ffe1ea399637aed5d",
"zh:725d6633fb92069bce53cb8b0f3b4d4a1fb4c0a336b138f62096dc2f7d4c2155",
"zh:8003646cc7caaa48841e802570626fd5cc8ad1bb2a341351ccf996eae62e88cb",
"zh:945f1f70842d04192626ae8e78372e48d16808d5104563bce32915c95236d820",
"zh:a0d8a25f8d84e78c3cfd5691f71c48f805ad38dab0a6a33f4d8e5cfc981b9cd9",
"zh:a3ba46c09233c4b77b63807654083385cc865e650bbb6274d8768bb18ff01508",
"zh:a80b7190ed733b9de6f3cfb55e82234457f51bb36bdcc11277a7623a47155cb4",
"zh:ba3f6f61deafaae1de92c17e924c7ef157ca0db2d5e14ae637a3a63bb1aeac9f",
"zh:c7b9790c722e597dc4e3d59bc9b510f364b3a522b70cd58727da09cd6adcf527",
"zh:f293b9ee146b2f22d79d4e53f0a1eb6bfdf8dca1d92bc39370a9df52046fdaa3",
]
}
provider "registry.terraform.io/timohirt/hetznerdns" { provider "registry.terraform.io/timohirt/hetznerdns" {
version = "2.2.0" version = "2.2.0"
constraints = ">= 2.2.0" constraints = ">= 2.2.0"

View file

@ -12,7 +12,6 @@ destroy: init
$(TF) destroy $(TFFLAGS) $(TF) destroy $(TFFLAGS)
lint: init lint: init
$(VALIDATE) k3s_cluster_v2
$(VALIDATE) . $(VALIDATE) .
init: keys/id_terraform init: keys/id_terraform
@ -27,9 +26,3 @@ import:
keys/id_terraform: keys/id_terraform:
echo "No private key found! Generating Terraform SSH Keys." echo "No private key found! Generating Terraform SSH Keys."
./scripts/bootstrap-keys.sh ./scripts/bootstrap-keys.sh
kubeconfig: keys/id_terraform
scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i keys/id_terraform root@`terraform output cluster_public_ip`:/etc/rancher/k3s/k3s.yaml ./kubeconfig.yaml
sed -i "s/127.0.0.1/`terraform output cluster_public_ip`/g" ./kubeconfig.yaml
sed -i "s/default/`terraform output cluster_name`/g" ./kubeconfig.yaml

21
dns.tf
View file

@ -25,7 +25,7 @@ resource "hetznerdns_record" "apricote_de_ns" {
resource "hetznerdns_record" "listory" { resource "hetznerdns_record" "listory" {
zone_id = hetznerdns_zone.apricote_de.id zone_id = hetznerdns_zone.apricote_de.id
name = "listory" name = "listory"
value = "c2.apricote.de" value = "listory.c3-ing.apricote.de."
type = "CNAME" type = "CNAME"
ttl = 60 ttl = 60
} }
@ -33,7 +33,24 @@ resource "hetznerdns_record" "listory" {
resource "hetznerdns_record" "gitea" { resource "hetznerdns_record" "gitea" {
zone_id = hetznerdns_zone.apricote_de.id zone_id = hetznerdns_zone.apricote_de.id
name = "gitea" name = "gitea"
value = "c2.apricote.de" value = "gitea.c3-ing.apricote.de."
type = "CNAME"
ttl = 60
}
resource "hetznerdns_record" "tandoor" {
zone_id = hetznerdns_zone.apricote_de.id
name = "tandoor"
value = "tandoor.c3-ing.apricote.de."
type = "CNAME"
ttl = 60
}
resource "hetznerdns_record" "grafana" {
zone_id = hetznerdns_zone.apricote_de.id
name = "grafana"
value = "grafana.c3-ing.apricote.de."
type = "CNAME" type = "CNAME"
ttl = 60 ttl = 60
} }

45
home_cloud_v3.tf Normal file
View file

@ -0,0 +1,45 @@
# kube api
data "hcloud_load_balancer" "c3_api" {
# LB is created and managed by cluster-api-provider-hetzner
name = "home-cloud-v3-mgtqc-kube-apiserver-4tbd8"
}
resource "hetznerdns_record" "c3_api_a" {
zone_id = hetznerdns_zone.apricote_de.id
name = "c3"
value = data.hcloud_load_balancer.c3_api.ipv4
type = "A"
ttl = 60
}
resource "hetznerdns_record" "c3_api_aaaa" {
zone_id = hetznerdns_zone.apricote_de.id
name = "c3"
value = data.hcloud_load_balancer.c3_api.ipv6
type = "AAAA"
ttl = 60
}
# ingress
data "hcloud_load_balancer" "c3_ingress" {
# LB is created and managed by hccm
name = "home-cloud-v3-traefik"
}
resource "hetznerdns_record" "c3_ingress_a" {
zone_id = hetznerdns_zone.apricote_de.id
name = "*.c3-ing"
value = data.hcloud_load_balancer.c3_ingress.ipv4
type = "A"
ttl = 60
}
resource "hetznerdns_record" "c3_ingress_aaaa" {
zone_id = hetznerdns_zone.apricote_de.id
name = "*.c3-ing"
value = data.hcloud_load_balancer.c3_ingress.ipv6
type = "AAAA"
ttl = 60
}

View file

@ -1,2 +0,0 @@
This module unfortunetly broke when I updated the CA cert in k3s and now
I can't get the state to work with terraform.

View file

@ -1,30 +0,0 @@
resource "hcloud_server" "agents" {
count = var.compute_count
name = "k3s-agent-${count.index}"
image = data.hcloud_image.ubuntu.name
server_type = var.compute_server_type
location = var.server_location
ssh_keys = [data.hcloud_ssh_key.default.id]
labels = {
provisioner = "terraform",
engine = "k3s",
node_type = "agent",
}
}
resource "hcloud_server_network" "agents_network" {
count = length(hcloud_server.agents)
server_id = hcloud_server.agents[count.index].id
subnet_id = hcloud_network_subnet.k3s_nodes.id
ip = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + var.control_count + count.index)
}
resource "hcloud_load_balancer_target" "ingress" {
count = var.compute_count
type = "server"
load_balancer_id = hcloud_load_balancer.k3s.id
server_id = hcloud_server.agents[count.index].id
use_private_ip = true
}

View file

@ -1,66 +0,0 @@
provider "github" {
owner = var.github_owner
token = var.github_token
}
# SSH
locals {
known_hosts = "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg="
}
resource "tls_private_key" "main" {
algorithm = "ECDSA"
ecdsa_curve = "P256"
}
# Flux
data "flux_install" "main" {
target_path = var.target_path
version = var.flux_version
}
data "flux_sync" "main" {
target_path = var.target_path
url = "ssh://git@github.com/${var.github_owner}/${var.repository_name}.git"
branch = var.branch
}
# GitHub
resource "github_repository" "main" {
name = var.repository_name
visibility = var.repository_visibility
auto_init = true
}
resource "github_branch_default" "main" {
repository = github_repository.main.name
branch = var.branch
}
resource "github_repository_deploy_key" "main" {
title = "staging-cluster"
repository = github_repository.main.name
key = tls_private_key.main.public_key_openssh
read_only = true
}
resource "github_repository_file" "install" {
repository = github_repository.main.name
file = data.flux_install.main.path
content = data.flux_install.main.content
branch = var.branch
}
resource "github_repository_file" "sync" {
repository = github_repository.main.name
file = data.flux_sync.main.path
content = data.flux_sync.main.content
branch = var.branch
}
resource "github_repository_file" "kustomize" {
repository = github_repository.main.name
file = data.flux_sync.main.kustomize_path
content = data.flux_sync.main.kustomize_content
branch = var.branch
}

View file

@ -1,95 +0,0 @@
data "hcloud_ssh_key" "default" {
name = "default"
}
resource "hcloud_network" "k3s" {
name = "k3s-network"
ip_range = "10.0.0.0/8"
}
resource "hcloud_network_subnet" "k3s_nodes" {
type = "cloud"
network_id = hcloud_network.k3s.id
network_zone = "eu-central"
ip_range = "10.254.1.0/24"
}
resource "hcloud_network_subnet" "lb" {
type = "cloud"
network_id = hcloud_network.k3s.id
network_zone = "eu-central"
ip_range = "10.254.2.0/24"
}
data "hcloud_image" "ubuntu" {
name = var.server_image
}
### Loadbalancer
resource "hcloud_load_balancer" "k3s" {
name = "k3s"
load_balancer_type = var.load_balancer_type
location = var.server_location
}
resource "hcloud_load_balancer_network" "k3s" {
load_balancer_id = hcloud_load_balancer.k3s.id
subnet_id = hcloud_network_subnet.lb.id
}
resource "hcloud_rdns" "k3s_ipv4" {
load_balancer_id = hcloud_load_balancer.k3s.id
ip_address = hcloud_load_balancer.k3s.ipv4
dns_ptr = var.domain
}
resource "hcloud_rdns" "k3s_ipv6" {
load_balancer_id = hcloud_load_balancer.k3s.id
ip_address = hcloud_load_balancer.k3s.ipv6
dns_ptr = var.domain
}
### LB Ingress
resource "hcloud_load_balancer_service" "ingress_https" {
load_balancer_id = hcloud_load_balancer.k3s.id
protocol = "tcp"
listen_port = 443
destination_port = 32443
}
resource "hcloud_load_balancer_service" "ingress_http" {
load_balancer_id = hcloud_load_balancer.k3s.id
protocol = "tcp"
listen_port = 80
destination_port = 32080
}
### Domain
resource "hetznerdns_record" "ipv4" {
zone_id = var.dns_zone_id
name = var.domain
value = hcloud_load_balancer.k3s.ipv4
type = "A"
ttl = 60
}
resource "hetznerdns_record" "ipv6" {
zone_id = var.dns_zone_id
name = var.domain
value = hcloud_load_balancer.k3s.ipv6
type = "AAAA"
ttl = 60
}
resource "hetznerdns_record" "wildcard" {
# *.domain CNAME domain
zone_id = var.dns_zone_id
name = "*"
value = var.domain
type = "CNAME"
ttl = 60
}

View file

@ -1,39 +0,0 @@
resource "hcloud_server" "control_planes" {
count = var.control_count
name = "k3s-control-plane-${count.index}"
image = data.hcloud_image.ubuntu.name
server_type = var.control_server_type
location = var.server_location
ssh_keys = [data.hcloud_ssh_key.default.id]
labels = {
provisioner = "terraform",
engine = "k3s",
node_type = "control-plane"
}
}
resource "hcloud_server_network" "control_planes" {
count = var.control_count
subnet_id = hcloud_network_subnet.k3s_nodes.id
server_id = hcloud_server.control_planes[count.index].id
ip = cidrhost(hcloud_network_subnet.k3s_nodes.ip_range, 1 + count.index)
}
# LB
resource "hcloud_load_balancer_service" "api" {
load_balancer_id = hcloud_load_balancer.k3s.id
protocol = "tcp"
listen_port = 6443
destination_port = 6443
}
resource "hcloud_load_balancer_target" "api" {
count = var.control_count
type = "server"
load_balancer_id = hcloud_load_balancer.k3s.id
server_id = hcloud_server.control_planes[count.index].id
use_private_ip = true
}

View file

@ -1,98 +0,0 @@
variable "name" {
type = string
}
variable "server_image" {
type = string
# With ubuntu-20.04 k3s crashes on start (v1.17.4+k3s1)
default = "ubuntu-18.04"
}
variable "server_location" {
type = string
}
variable "control_server_type" {
type = string
default = "cx21"
}
variable "compute_server_type" {
type = string
default = "cpx21"
}
variable "control_count" {
description = "Number of control plane nodes."
default = 3
}
variable "compute_count" {
type = number
default = 1
}
variable "load_balancer_type" {
type = string
default = "lb11"
}
variable "domain" {
type = string
}
variable "dns_zone_id" {
type = string
}
variable "install_k3s_version" {
type = string
default = "v1.22.4+k3s1"
}
variable "ssh_key" {
description = "SSH public Key content needed to provision the instances."
type = string
}
variable "hcloud_ccm_token" {
description = "HCloud API Token used by the hcloud-cloud-controller-manager"
type = string
sensitive = true
}
## Flux
variable "github_owner" {
type = string
}
variable "github_token" {
type = string
sensitive = true
}
variable "github_token_flux_notifications" {
type = string
sensitive = true
}
variable "repository_name" {
type = string
}
variable "branch" {
type = string
}
variable "repository_visibility" {
type = string
}
variable "flux_version" {
type = string
}
variable "target_path" {
type = string
description = "Relative path to the Git repository root where Flux manifests are committed."
}

View file

@ -1,40 +0,0 @@
terraform {
required_version = ">= 1.0"
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
}
hetznerdns = {
source = "timohirt/hetznerdns"
}
tls = {
source = "hashicorp/tls"
}
template = {
source = "hashicorp/template"
}
random = {
source = "hashicorp/random"
}
null = {
source = "hashicorp/null"
}
github = {
source = "integrations/github"
}
kubernetes = {
source = "hashicorp/kubernetes"
}
kubectl = {
source = "gavinbunney/kubectl"
version = ">= 1.10.0"
}
flux = {
source = "fluxcd/flux"
version = ">= 0.10.0"
}
}
}

32
main.tf
View file

@ -2,35 +2,3 @@ locals {
cluster_name = "home-cloud" cluster_name = "home-cloud"
} }
module "k3s_cluster_v2" {
source = "./k3s_cluster_v2"
name = local.cluster_name
server_image = "ubuntu-20.04"
server_location = "nbg1"
control_server_type = "cx21"
compute_server_type = "cpx31"
load_balancer_type = "lb11"
install_k3s_version = "v1.22.4+k3s1"
control_count = 1
compute_count = 1
domain = "c2.apricote.de"
dns_zone_id = hetznerdns_zone.apricote_de.id
ssh_key = file("~/.ssh/id_rsa.pub")
hcloud_ccm_token = var.hcloud_ccm_token
## Flux
github_owner = "apricote"
github_token = var.github_token
github_token_flux_notifications = var.github_token_flux_notifications
repository_name = "home-cloud-flux-v2"
branch = "main"
repository_visibility = "private"
target_path = ""
flux_version = "v0.36.0"
providers = {
hcloud = hcloud
hetznerdns = hetznerdns
}
}

View file

@ -1,25 +1,3 @@
variable "hcloud_csi_driver_token" {
type = string
sensitive = true
}
variable "hcloud_ccm_token" {
type = string
sensitive = true
}
variable "github_token" {
description = "Github Personal Access Token that is used by Terraform"
type = string
sensitive = true
}
variable "github_token_flux_notifications" {
description = "GH PAT used by flux for notifications"
type = string
sensitive = true
}
variable "listory_token" { variable "listory_token" {
description = "Listory API Token" description = "Listory API Token"
type = string type = string