diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt index 0c34580..0224696 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt @@ -48,16 +48,19 @@ fun Prov.installK3s(k3sConfig: K3sConfig) = task { createDirs(k3sAutomatedManifestsDir, sudo = true) createDirs(k3sManualManifestsDir, sudo = true) var k3sConfigFileName = "config" + var metallbConfigFileName = "metallb-config" var k3sConfigMap: Map = mapOf( "loopback_ipv4" to k3sConfig.loopback.ipv4, "node_ipv4" to k3sConfig.node.ipv4, "tls_name" to k3sConfig.fqdn ) if (k3sConfig.isDualStack()) { k3sConfigFileName += ".dual.template.yaml" + metallbConfigFileName += ".dual.template.yaml" k3sConfigMap = k3sConfigMap.plus("node_ipv6" to k3sConfig.node.ipv6!!) .plus("loopback_ipv6" to k3sConfig.loopback.ipv6!!) } else { k3sConfigFileName += ".ipv4.template.yaml" + metallbConfigFileName += ".ipv4.template.yaml" } createFileFromResourceTemplate( k3sConfigFile, @@ -75,6 +78,33 @@ fun Prov.installK3s(k3sConfig: K3sConfig) = task { sudo = true ) cmd("INSTALL_K3S_CHANNEL=latest k3s-install.sh") + + // metallb + createFileFromResource( + k3sManualManifestsDir + "metallb-namespace.yaml", + "metallb-namespace.yaml", + k3sResourcePath, + sudo = true + ) + createFileFromResource( + k3sManualManifestsDir + "metallb-manifest.yaml", + "metallb-0.10.2-manifest.yaml", + k3sResourcePath, + sudo = true + ) + createFileFromResourceTemplate( + k3sManualManifestsDir + "metallb-config.yaml", + metallbConfigFileName, + k3sResourcePath, + k3sConfigMap, + "644", + sudo = true + ) + cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-namespace.yaml", sudo = true) + cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-manifest.yaml", sudo = true) + cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-config.yaml", sudo = true) + + // traefic if (k3sConfig.isDualStack()) { // see https://github.com/k3s-io/k3s/discussions/5003 createFileFromResource( diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.dual.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.dual.template.yaml index d2b5753..6b81cd3 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.dual.template.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.dual.template.yaml @@ -1,9 +1,11 @@ default-local-storage-path: /var tls-san: ${tls_name} +kube-proxy-arg proxy-mode: ipvs disable-network-policy: true disable-cloud-controller: true disable: - traefik + - servicelb cluster-cidr: - 10.42.0.0/16 - fd42::/48 diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.ipv4.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.ipv4.template.yaml index 4773cdb..83fdb25 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.ipv4.template.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.ipv4.template.yaml @@ -1,7 +1,10 @@ default-local-storage-path: /var tls-san: ${tls_name} +kube-proxy-arg proxy-mode: ipvs disable-network-policy: true disable-cloud-controller: true +disable: + - servicelb cluster-cidr: - 10.42.0.0/16 service-cidr: diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-0.10.2-manifest.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-0.10.2-manifest.yaml new file mode 100644 index 0000000..9d6b683 --- /dev/null +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-0.10.2-manifest.yaml @@ -0,0 +1,480 @@ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: controller +spec: + allowPrivilegeEscalation: false + allowedCapabilities: [] + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + hostIPC: false + hostNetwork: false + hostPID: false + privileged: false + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: speaker +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_RAW + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + hostIPC: false + hostNetwork: true + hostPID: false + hostPorts: + - max: 7472 + min: 7472 + - max: 7946 + min: 7946 + privileged: true + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - services/status + verbs: + - update +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resourceNames: + - controller + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - '' + resources: + - services + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - pods + verbs: + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - secrets + verbs: + - create +- apiGroups: + - '' + resources: + - secrets + resourceNames: + - memberlist + verbs: + - list +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - controller + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-lister +subjects: +- kind: ServiceAccount + name: speaker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: controller +subjects: +- kind: ServiceAccount + name: controller +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - --port=7472 + - --config=config + - --log-level=info + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: METALLB_ML_BIND_ADDR + valueFrom: + fieldRef: + fieldPath: status.podIP + # needed when another software is also using memberlist / port 7946 + # when changing this default you also need to update the container ports definition + # and the PodSecurityPolicy hostPorts definition + #- name: METALLB_ML_BIND_PORT + # value: "7946" + - name: METALLB_ML_LABELS + value: "app=metallb,component=speaker" + - name: METALLB_ML_SECRET_KEY + valueFrom: + secretKeyRef: + name: memberlist + key: secretkey + image: quay.io/metallb/speaker:v0.12.1 + name: speaker + ports: + - containerPort: 7472 + name: monitoring + - containerPort: 7946 + name: memberlist-tcp + - containerPort: 7946 + name: memberlist-udp + protocol: UDP + livenessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_RAW + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 2 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + containers: + - args: + - --port=7472 + - --config=config + - --log-level=info + env: + - name: METALLB_ML_SECRET_NAME + value: memberlist + - name: METALLB_DEPLOYMENT + value: controller + image: quay.io/metallb/controller:v0.12.1 + name: controller + ports: + - containerPort: 7472 + name: monitoring + livenessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.dual.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.dual.template.yaml new file mode 100644 index 0000000..2bb9135 --- /dev/null +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.dual.template.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - ${node_ipv4}/32 + - ${node_ipv6}/128 \ No newline at end of file diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.ipv4.template.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.ipv4.template.yaml new file mode 100644 index 0000000..f56534f --- /dev/null +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.ipv4.template.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - ${node_ipv4}/32 \ No newline at end of file diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-namespace.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-namespace.yaml new file mode 100644 index 0000000..003269b --- /dev/null +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system + labels: + app: metallb diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/traefik.yaml b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/traefik.yaml index d9cb89a..6fceff2 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/traefik.yaml +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/traefik.yaml @@ -19,6 +19,8 @@ spec: valuesContent: |- service: spec: + type: LoadBalancer + externalTrafficPolicy: Local ipFamilyPolicy: RequireDualStack rbac: enabled: true