introduce metallb

src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt

@@ -48,16 +48,19 @@ fun Prov.installK3s(k3sConfig: K3sConfig) = task {
     createDirs(k3sAutomatedManifestsDir, sudo = true)
     createDirs(k3sManualManifestsDir, sudo = true)
     var k3sConfigFileName = "config"
+    var metallbConfigFileName = "metallb-config"
     var k3sConfigMap: Map<String, String> = mapOf(
         "loopback_ipv4" to k3sConfig.loopback.ipv4,
         "node_ipv4" to k3sConfig.node.ipv4, "tls_name" to k3sConfig.fqdn
     )
     if (k3sConfig.isDualStack()) {
         k3sConfigFileName += ".dual.template.yaml"
+        metallbConfigFileName += ".dual.template.yaml"
         k3sConfigMap = k3sConfigMap.plus("node_ipv6" to k3sConfig.node.ipv6!!)
             .plus("loopback_ipv6" to k3sConfig.loopback.ipv6!!)
     } else {
         k3sConfigFileName += ".ipv4.template.yaml"
+        metallbConfigFileName += ".ipv4.template.yaml"
     }
     createFileFromResourceTemplate(
         k3sConfigFile,
@@ -75,6 +78,33 @@ fun Prov.installK3s(k3sConfig: K3sConfig) = task {
         sudo = true
     )
     cmd("INSTALL_K3S_CHANNEL=latest k3s-install.sh")
+
+    // metallb
+    createFileFromResource(
+        k3sManualManifestsDir + "metallb-namespace.yaml",
+        "metallb-namespace.yaml",
+        k3sResourcePath,
+        sudo = true
+    )
+    createFileFromResource(
+        k3sManualManifestsDir + "metallb-manifest.yaml",
+        "metallb-0.10.2-manifest.yaml",
+        k3sResourcePath,
+        sudo = true
+    )
+    createFileFromResourceTemplate(
+        k3sManualManifestsDir + "metallb-config.yaml",
+        metallbConfigFileName,
+        k3sResourcePath,
+        k3sConfigMap,
+        "644",
+        sudo = true
+    )
+    cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-namespace.yaml", sudo = true)
+    cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-manifest.yaml", sudo = true)
+    cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-config.yaml", sudo = true)
+
+    // traefic
     if (k3sConfig.isDualStack()) {
         // see https://github.com/k3s-io/k3s/discussions/5003
         createFileFromResource(

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.dual.template.yaml

@@ -1,9 +1,11 @@
 default-local-storage-path: /var
 tls-san: ${tls_name}
+kube-proxy-arg proxy-mode: ipvs
 disable-network-policy: true
 disable-cloud-controller: true
 disable:
   - traefik
+  - servicelb
 cluster-cidr:
   - 10.42.0.0/16
   - fd42::/48

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.ipv4.template.yaml

@@ -1,7 +1,10 @@
 default-local-storage-path: /var
 tls-san: ${tls_name}
+kube-proxy-arg proxy-mode: ipvs
 disable-network-policy: true
 disable-cloud-controller: true
+disable:
+  - servicelb
 cluster-cidr:
   - 10.42.0.0/16
 service-cidr:

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-0.10.2-manifest.yaml

@@ -0,0 +1,480 @@
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: controller
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities: []
+  allowedHostPaths: []
+  defaultAddCapabilities: []
+  defaultAllowPrivilegeEscalation: false
+  fsGroup:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  hostIPC: false
+  hostNetwork: false
+  hostPID: false
+  privileged: false
+  readOnlyRootFilesystem: true
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  volumes:
+  - configMap
+  - secret
+  - emptyDir
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities:
+  - NET_RAW
+  allowedHostPaths: []
+  defaultAddCapabilities: []
+  defaultAllowPrivilegeEscalation: false
+  fsGroup:
+    rule: RunAsAny
+  hostIPC: false
+  hostNetwork: true
+  hostPID: false
+  hostPorts:
+  - max: 7472
+    min: 7472
+  - max: 7946
+    min: 7946
+  privileged: true
+  readOnlyRootFilesystem: true
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+  - configMap
+  - secret
+  - emptyDir
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resourceNames:
+  - controller
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: ["discovery.k8s.io"]
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resourceNames:
+  - speaker
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: pod-lister
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - create
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  resourceNames:
+  - memberlist
+  verbs:
+  - list
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  resourceNames:
+  - controller
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: pod-lister
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-lister
+subjects:
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: controller
+subjects:
+- kind: ServiceAccount
+  name: controller
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: metallb
+    component: speaker
+  name: speaker
+  namespace: metallb-system
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '7472'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: speaker
+    spec:
+      containers:
+      - args:
+        - --port=7472
+        - --config=config
+        - --log-level=info
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: METALLB_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: METALLB_ML_BIND_ADDR
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        # needed when another software is also using memberlist / port 7946
+        # when changing this default you also need to update the container ports definition
+        # and the PodSecurityPolicy hostPorts definition
+        #- name: METALLB_ML_BIND_PORT
+        #  value: "7946"
+        - name: METALLB_ML_LABELS
+          value: "app=metallb,component=speaker"
+        - name: METALLB_ML_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: memberlist
+              key: secretkey
+        image: quay.io/metallb/speaker:v0.12.1
+        name: speaker
+        ports:
+        - containerPort: 7472
+          name: monitoring
+        - containerPort: 7946
+          name: memberlist-tcp
+        - containerPort: 7946
+          name: memberlist-udp
+          protocol: UDP
+        livenessProbe:
+          httpGet:
+            path: /metrics
+            port: monitoring
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: /metrics
+            port: monitoring
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 3
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_RAW
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 2
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Exists
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: metallb
+    component: controller
+  name: controller
+  namespace: metallb-system
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '7472'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: controller
+    spec:
+      containers:
+      - args:
+        - --port=7472
+        - --config=config
+        - --log-level=info
+        env:
+        - name: METALLB_ML_SECRET_NAME
+          value: memberlist
+        - name: METALLB_DEPLOYMENT
+          value: controller
+        image: quay.io/metallb/controller:v0.12.1
+        name: controller
+        ports:
+        - containerPort: 7472
+          name: monitoring
+        livenessProbe:
+          httpGet:
+            path: /metrics
+            port: monitoring
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: /metrics
+            port: monitoring
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 3
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        fsGroup: 65534
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.dual.template.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - ${node_ipv4}/32
+      - ${node_ipv6}/128

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.ipv4.template.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - ${node_ipv4}/32

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+  labels:
+    app: metallb

src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/traefik.yaml

@@ -19,6 +19,8 @@ spec:
   valuesContent: |-
     service:
       spec:
+        type: LoadBalancer
+        externalTrafficPolicy: Local
         ipFamilyPolicy: RequireDualStack
     rbac:
       enabled: true