From fbac80849103eb747d25867f3bf907fc0cad3464 Mon Sep 17 00:00:00 2001 From: jem Date: Sat, 29 Jan 2022 21:32:16 +0100 Subject: [PATCH] enabled dualstack on traefic --- .../provs/server/infrastructure/K3s.kt | 36 +++++++++----- .../k3s/config.dual.template.yaml | 2 + .../provs/infrastructure/k3s/traefic.yaml | 47 +++++++++++++++++++ 3 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt index 28a7237..c04dc08 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt @@ -7,24 +7,24 @@ import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.* private const val k3sResourcePath = "org/domaindrivenarchitecture/provs/infrastructure/k3s/" private const val k3sManifestsDir = "/etc/rancher/k3s/manifests/" -private const val k3sConfigFile = "/etc/rancher/k3s/config.yaml" -private const val k3sAppleFile = k3sManifestsDir + "apple.yaml" +private const val k3sConfig = "/etc/rancher/k3s/config.yaml" +private const val k3sTraeficWorkaround = "/var/lib/rancher/k3s/server/manifests/traefik-workaround.yaml" +private const val k3sApple = k3sManifestsDir + "apple.yaml" private const val certManagerDeployment = k3sManifestsDir + "certmanager.yaml" private const val certManagerIssuer = k3sManifestsDir + "issuer.yaml" - -private const val k3sInstallFile = "/usr/local/bin/k3s-install.sh" +private const val k3sInstall = "/usr/local/bin/k3s-install.sh" enum class CertManagerEndPoint { STAGING, PROD } fun Prov.testConfigExists(): Boolean { - return fileExists(k3sConfigFile) + return fileExists(k3sConfig) } fun Prov.deprovisionK3sInfra() = task { - deleteFile(k3sInstallFile, sudo = true) - deleteFile(k3sAppleFile, sudo = true) + deleteFile(k3sInstall, sudo = true) + deleteFile(k3sApple, sudo = true) deleteFile(certManagerDeployment, sudo = true) deleteFile(certManagerIssuer, sudo = true) cmd("k3s-uninstall.sh") @@ -36,7 +36,7 @@ fun Prov.deprovisionK3sInfra() = task { * If tlsHost is specified, then tls (if configured) also applies to the specified host. */ fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: String, loopbackIpv6: String, - nodeIpv6: String? = null, tlsHost: String? = null) = task { + nodeIpv6: String? = null) = task { val isDualStack = nodeIpv6?.isNotEmpty() ?: false if (!testConfigExists()) { createDirs(k3sManifestsDir, sudo = true) @@ -50,7 +50,7 @@ fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: Stri k3sConfigFileName += ".ipv4.template.yaml" } createFileFromResourceTemplate( - k3sConfigFile, + k3sConfig, k3sConfigFileName, k3sResourcePath, k3sConfigMap, @@ -58,13 +58,25 @@ fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: Stri sudo = true ) createFileFromResource( - k3sInstallFile, + k3sInstall, "k3s-install.sh", k3sResourcePath, "755", sudo = true ) cmd("k3s-install.sh") + if(isDualStack) { + // see https://github.com/k3s-io/k3s/discussions/5003 + createFileFromResource( + k3sTraeficWorkaround, + "traefic.yaml", + k3sResourcePath, + "644", + sudo = true + ) + } else { + ProvResult(true) + } } else { ProvResult(true) } @@ -96,14 +108,14 @@ fun Prov.provisionK3sCertManager(endpoint: CertManagerEndPoint) = task { fun Prov.provisionK3sApple(fqdn: String, endpoint: CertManagerEndPoint) = task { createFileFromResourceTemplate( - k3sAppleFile, + k3sApple, "apple.template.yaml", k3sResourcePath, mapOf("fqdn" to fqdn, "issuer_name" to endpoint.name.lowercase()), "644", sudo = true ) - cmd("kubectl apply -f $k3sAppleFile", sudo = true) + cmd("kubectl apply -f $k3sApple", sudo = true) repeatTaskUntilSuccess(10, 10) { cmd("kubectl apply -f $certManagerIssuer", sudo = true) diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml index bd8073d..9f33abe 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml @@ -1,6 +1,8 @@ default-local-storage-path: /var tls-san: ${tls_name} disable-network-policy: true +disable: + - traefik cluster-cidr: - 10.42.0.0/16 - fd42::/48 diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml new file mode 100644 index 0000000..7a9c9ed --- /dev/null +++ b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: traefik-crd + namespace: kube-system +spec: + chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-10.3.001.tgz +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: traefik + namespace: kube-system +spec: + chart: https://%{KUBERNETES_API}%/static/charts/traefik-10.3.001.tgz + set: + global.systemDefaultRegistry: "" + valuesContent: |- + service: + spec: + ipFamilyPolicy: RequireDualStack + rbac: + enabled: true + ports: + websecure: + tls: + enabled: true + podAnnotations: + prometheus.io/port: "8082" + prometheus.io/scrape: "true" + providers: + kubernetesIngress: + publishedService: + enabled: true + priorityClassName: "system-cluster-critical" + image: + name: "rancher/mirrored-library-traefik" + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" \ No newline at end of file