diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/domain/k3s/K3sService.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/domain/k3s/K3sService.kt index 89c582e..2e2eaef 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/domain/k3s/K3sService.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/domain/k3s/K3sService.kt @@ -2,6 +2,7 @@ package org.domaindrivenarchitecture.provs.server.domain.k3s import org.domaindrivenarchitecture.provs.framework.core.Prov import org.domaindrivenarchitecture.provs.configuration.domain.ConfigFileName +import org.domaindrivenarchitecture.provs.framework.core.ProvResult import org.domaindrivenarchitecture.provs.server.infrastructure.* import org.domaindrivenarchitecture.provs.server.infrastructure.k3s.getK3sConfig @@ -13,15 +14,16 @@ import org.domaindrivenarchitecture.provs.server.infrastructure.k3s.getK3sConfig fun Prov.provisionK3s(configFileName: ConfigFileName?) = task { val k3sConfig: K3sConfig = getK3sConfig(configFileName!!) - provisionNetwork(loopbackIpv4 = k3sConfig.loopback.ipv4, loopbackIpv6 = k3sConfig.loopback.ipv6) + provisionNetwork(k3sConfig) if (k3sConfig.reprovision && testConfigExists()) { deprovisionK3sInfra() } - provisionK3sInfra( - tlsName = k3sConfig.fqdn, nodeIpv4 = k3sConfig.node.ipv4, nodeIpv6 = k3sConfig.node.ipv6, - loopbackIpv4 = k3sConfig.loopback.ipv4, loopbackIpv6 = k3sConfig.loopback.ipv6 - ) - - provisionK3sCertManager(k3sConfig.letsencryptEndpoint) - provisionK3sApple(k3sConfig.fqdn, k3sConfig.letsencryptEndpoint) + provisionK3sInfra(k3sConfig) + if (k3sConfig.certmanager != null) { + provisionK3sCertManager(k3sConfig.certmanager) + } + if (k3sConfig.apple != null && k3sConfig.apple) { + provisionK3sApple(k3sConfig.fqdn, k3sConfig.certmanager!!.letsencryptEndpoint) + } + ProvResult(true) } diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt index 828171d..2bd0c4f 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt @@ -4,23 +4,22 @@ import org.domaindrivenarchitecture.provs.framework.core.Prov import org.domaindrivenarchitecture.provs.framework.core.ProvResult import org.domaindrivenarchitecture.provs.framework.core.repeatTaskUntilSuccess import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.* +import org.domaindrivenarchitecture.provs.server.domain.CertmanagerEndpoint +import org.domaindrivenarchitecture.provs.server.domain.k3s.Certmanager +import org.domaindrivenarchitecture.provs.server.domain.k3s.K3sConfig private const val k3sResourcePath = "org/domaindrivenarchitecture/provs/infrastructure/k3s/" private const val k3sManualManifestsDir = "/etc/rancher/k3s/manifests/" private const val k3sAutomatedManifestsDir = "/var/lib/rancher/k3s/server/manifests/" -private const val k3sConfig = "/etc/rancher/k3s/config.yaml" -private const val k3sTraeficWorkaround = "/var/lib/rancher/k3s/server/manifests/traefik-workaround.yaml" -private const val k3sApple = k3sAutomatedManifestsDir + "apple.yaml" -private const val certManagerDeployment = k3sAutomatedManifestsDir + "certmanager.yaml" +private const val k3sConfigFile = "/etc/rancher/k3s/config.yaml" +private const val k3sTraeficWorkaround = k3sManualManifestsDir + "traefik.yaml" +private const val certManagerDeployment = k3sManualManifestsDir + "certmanager.yaml" private const val certManagerIssuer = k3sManualManifestsDir + "issuer.yaml" +private const val k3sApple = k3sManualManifestsDir + "apple.yaml" private const val k3sInstall = "/usr/local/bin/k3s-install.sh" -enum class CertManagerEndPoint { - STAGING, PROD -} - fun Prov.testConfigExists(): Boolean { - return fileExists(k3sConfig) + return fileExists(k3sConfigFile) } fun Prov.deprovisionK3sInfra() = task { @@ -36,24 +35,24 @@ fun Prov.deprovisionK3sInfra() = task { * If docker is true, then docker will be installed (may conflict if docker is already existing) and k3s will be installed with docker option. * If tlsHost is specified, then tls (if configured) also applies to the specified host. */ -fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: String, loopbackIpv6: String?, - nodeIpv6: String? = null) = task { - val isDualStack = nodeIpv6 != null && loopbackIpv6 != null +fun Prov.provisionK3sInfra(k3sConfig: K3sConfig) = task { if (!testConfigExists()) { createDirs(k3sAutomatedManifestsDir, sudo = true) createDirs(k3sManualManifestsDir, sudo = true) var k3sConfigFileName = "config" - var k3sConfigMap: Map = mapOf("loopback_ipv4" to loopbackIpv4, - "node_ipv4" to nodeIpv4, "tls_name" to tlsName) - if (isDualStack) { + var k3sConfigMap: Map = mapOf( + "loopback_ipv4" to k3sConfig.loopback.ipv4, + "node_ipv4" to k3sConfig.node.ipv4, "tls_name" to k3sConfig.fqdn + ) + if (k3sConfig.isDualStack()) { k3sConfigFileName += ".dual.template.yaml" - k3sConfigMap = k3sConfigMap.plus("node_ipv6" to nodeIpv6!!) - .plus("loopback_ipv6" to loopbackIpv6!!) + k3sConfigMap = k3sConfigMap.plus("node_ipv6" to k3sConfig.node.ipv6!!) + .plus("loopback_ipv6" to k3sConfig.loopback.ipv6!!) } else { k3sConfigFileName += ".ipv4.template.yaml" } createFileFromResourceTemplate( - k3sConfig, + k3sConfigFile, k3sConfigFileName, k3sResourcePath, k3sConfigMap, @@ -68,7 +67,7 @@ fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: Stri sudo = true ) cmd("k3s-install.sh") - if(isDualStack) { + if (k3sConfig.isDualStack()) { // see https://github.com/k3s-io/k3s/discussions/5003 createFileFromResource( k3sTraeficWorkaround, @@ -77,6 +76,7 @@ fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: Stri "644", sudo = true ) + cmd ("kubectl apply -f $k3sTraeficWorkaround", sudo = true) } else { ProvResult(true) } @@ -86,7 +86,7 @@ fun Prov.provisionK3sInfra(tlsName: String, nodeIpv4: String, loopbackIpv4: Stri } -fun Prov.provisionK3sCertManager(endpoint: CertManagerEndPoint) = task { +fun Prov.provisionK3sCertManager(certmanager: Certmanager) = task { createFileFromResource( certManagerDeployment, "cert-manager.yaml", @@ -94,11 +94,15 @@ fun Prov.provisionK3sCertManager(endpoint: CertManagerEndPoint) = task { "644", sudo = true ) + cmd ("kubectl apply -f $certManagerDeployment", sudo = true) createFileFromResourceTemplate( certManagerIssuer, "le-issuer.template.yaml", k3sResourcePath, - mapOf("endpoint" to endpoint.name.lowercase()), + mapOf( + "endpoint" to certmanager.letsencryptEndpoint.name.lowercase(), + "email" to certmanager.email + ), "644", sudo = true ) @@ -107,7 +111,7 @@ fun Prov.provisionK3sCertManager(endpoint: CertManagerEndPoint) = task { } } -fun Prov.provisionK3sApple(fqdn: String, endpoint: CertManagerEndPoint) = task { +fun Prov.provisionK3sApple(fqdn: String, endpoint: CertmanagerEndpoint) = task { createFileFromResourceTemplate( k3sApple, "apple.template.yaml", @@ -116,4 +120,5 @@ fun Prov.provisionK3sApple(fqdn: String, endpoint: CertManagerEndPoint) = task { "644", sudo = true ) + cmd("kubectl apply -f $k3sApple", sudo = true) } diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml index 9f33abe..d2b5753 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.dual.template.yaml @@ -1,6 +1,7 @@ default-local-storage-path: /var tls-san: ${tls_name} disable-network-policy: true +disable-cloud-controller: true disable: - traefik cluster-cidr: diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.ipv4.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.ipv4.template.yaml index 434ef6d..4773cdb 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.ipv4.template.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/config.ipv4.template.yaml @@ -1,6 +1,7 @@ default-local-storage-path: /var tls-san: ${tls_name} disable-network-policy: true +disable-cloud-controller: true cluster-cidr: - 10.42.0.0/16 service-cidr: diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/le-issuer.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/le-issuer.template.yaml index 920eb5e..d4ab188 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/le-issuer.template.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/le-issuer.template.yaml @@ -4,7 +4,7 @@ metadata: name: ${endpoint} spec: acme: - email: admin@meissa-gmbh.de + email: ${email} server: https://acme-${endpoint}-v02.api.letsencrypt.org/directory privateKeySecretRef: name: ${endpoint} diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml index 7a9c9ed..11cc62c 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/infrastructure/k3s/traefic.yaml @@ -5,7 +5,7 @@ metadata: name: traefik-crd namespace: kube-system spec: - chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-10.3.001.tgz + chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-10.9.100.tgz --- apiVersion: helm.cattle.io/v1 kind: HelmChart @@ -13,7 +13,7 @@ metadata: name: traefik namespace: kube-system spec: - chart: https://%{KUBERNETES_API}%/static/charts/traefik-10.3.001.tgz + chart: https://%{KUBERNETES_API}%/static/charts/traefik-10.9.100.tgz set: global.systemDefaultRegistry: "" valuesContent: |- @@ -44,4 +44,4 @@ spec: effect: "NoSchedule" - key: "node-role.kubernetes.io/master" operator: "Exists" - effect: "NoSchedule" \ No newline at end of file + effect: "NoSchedule"