meissa/provs
7
1
Fork 0
Code Issues Pull requests Projects Releases 12 Packages Wiki Activity

introduce metallb

Browse source
This commit is contained in:
jerger 2022-03-11 17:51:03 +01:00
parent bd83b4a0f8
commit a9b74342a7
8 changed files with 548 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt
View file

@ -48,16 +48,19 @@ fun Prov.installK3s(k3sConfig: K3sConfig) = task {
createDirs(k3sAutomatedManifestsDir, sudo = true)
createDirs(k3sManualManifestsDir, sudo = true)
var k3sConfigFileName = "config"
var metallbConfigFileName = "metallb-config"
var k3sConfigMap: Map<String, String> = mapOf(
"loopback_ipv4" to k3sConfig.loopback.ipv4,
"node_ipv4" to k3sConfig.node.ipv4, "tls_name" to k3sConfig.fqdn
)
if (k3sConfig.isDualStack()) {
k3sConfigFileName += ".dual.template.yaml"
metallbConfigFileName += ".dual.template.yaml"
k3sConfigMap = k3sConfigMap.plus("node_ipv6" to k3sConfig.node.ipv6!!)
.plus("loopback_ipv6" to k3sConfig.loopback.ipv6!!)
} else {
k3sConfigFileName += ".ipv4.template.yaml"
metallbConfigFileName += ".ipv4.template.yaml"
}
createFileFromResourceTemplate(
k3sConfigFile,
@ -75,6 +78,33 @@ fun Prov.installK3s(k3sConfig: K3sConfig) = task {
sudo = true
)
cmd("INSTALL_K3S_CHANNEL=latest k3s-install.sh")
// metallb
createFileFromResource(
k3sManualManifestsDir + "metallb-namespace.yaml",
"metallb-namespace.yaml",
k3sResourcePath,
sudo = true
)
createFileFromResource(
k3sManualManifestsDir + "metallb-manifest.yaml",
"metallb-0.10.2-manifest.yaml",
k3sResourcePath,
sudo = true
)
createFileFromResourceTemplate(
k3sManualManifestsDir + "metallb-config.yaml",
metallbConfigFileName,
k3sResourcePath,
k3sConfigMap,
"644",
sudo = true
)
cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-namespace.yaml", sudo = true)
cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-manifest.yaml", sudo = true)
cmd("kubectl apply -f ${k3sManualManifestsDir}metallb-config.yaml", sudo = true)
// traefic
if (k3sConfig.isDualStack()) {
// see https://github.com/k3s-io/k3s/discussions/5003
createFileFromResource(

2
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.dual.template.yaml
View file

@ -1,9 +1,11 @@
default-local-storage-path: /var
tls-san: ${tls_name}
kube-proxy-arg proxy-mode: ipvs
disable-network-policy: true
disable-cloud-controller: true
disable:
- traefik
- servicelb
cluster-cidr:
- 10.42.0.0/16
- fd42::/48

3
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/config.ipv4.template.yaml
View file

@ -1,7 +1,10 @@
default-local-storage-path: /var
tls-san: ${tls_name}
kube-proxy-arg proxy-mode: ipvs
disable-network-policy: true
disable-cloud-controller: true
disable:
- servicelb
cluster-cidr:
- 10.42.0.0/16
service-cidr:

480
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-0.10.2-manifest.yaml Normal file
View file

@ -0,0 +1,480 @@
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
labels:
app: metallb
name: controller
spec:
allowPrivilegeEscalation: false
allowedCapabilities: []
allowedHostPaths: []
defaultAddCapabilities: []
defaultAllowPrivilegeEscalation: false
fsGroup:
ranges:
- max: 65535
min: 1
rule: MustRunAs
hostIPC: false
hostNetwork: false
hostPID: false
privileged: false
readOnlyRootFilesystem: true
requiredDropCapabilities:
- ALL
runAsUser:
ranges:
- max: 65535
min: 1
rule: MustRunAs
seLinux:
rule: RunAsAny
supplementalGroups:
ranges:
- max: 65535
min: 1
rule: MustRunAs
volumes:
- configMap
- secret
- emptyDir
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
labels:
app: metallb
name: speaker
spec:
allowPrivilegeEscalation: false
allowedCapabilities:
- NET_RAW
allowedHostPaths: []
defaultAddCapabilities: []
defaultAllowPrivilegeEscalation: false
fsGroup:
rule: RunAsAny
hostIPC: false
hostNetwork: true
hostPID: false
hostPorts:
- max: 7472
min: 7472
- max: 7946
min: 7946
privileged: true
readOnlyRootFilesystem: true
requiredDropCapabilities:
- ALL
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- configMap
- secret
- emptyDir
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: metallb
name: controller
namespace: metallb-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: metallb
name: speaker
namespace: metallb-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app: metallb
name: metallb-system:controller
rules:
- apiGroups:
- ''
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:
- services/status
verbs:
- update
- apiGroups:
- ''
resources:
- events
verbs:
- create
- patch
- apiGroups:
- policy
resourceNames:
- controller
resources:
- podsecuritypolicies
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app: metallb
name: metallb-system:speaker
rules:
- apiGroups:
- ''
resources:
- services
- endpoints
- nodes
verbs:
- get
- list
- watch
- apiGroups: ["discovery.k8s.io"]
resources:
- endpointslices
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:
- events
verbs:
- create
- patch
- apiGroups:
- policy
resourceNames:
- speaker
resources:
- podsecuritypolicies
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app: metallb
name: config-watcher
namespace: metallb-system
rules:
- apiGroups:
- ''
resources:
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app: metallb
name: pod-lister
namespace: metallb-system
rules:
- apiGroups:
- ''
resources:
- pods
verbs:
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app: metallb
name: controller
namespace: metallb-system
rules:
- apiGroups:
- ''
resources:
- secrets
verbs:
- create
- apiGroups:
- ''
resources:
- secrets
resourceNames:
- memberlist
verbs:
- list
- apiGroups:
- apps
resources:
- deployments
resourceNames:
- controller
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app: metallb
name: metallb-system:controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: metallb-system:controller
subjects:
- kind: ServiceAccount
name: controller
namespace: metallb-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app: metallb
name: metallb-system:speaker
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: metallb-system:speaker
subjects:
- kind: ServiceAccount
name: speaker
namespace: metallb-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app: metallb
name: config-watcher
namespace: metallb-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: config-watcher
subjects:
- kind: ServiceAccount
name: controller
- kind: ServiceAccount
name: speaker
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app: metallb
name: pod-lister
namespace: metallb-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: pod-lister
subjects:
- kind: ServiceAccount
name: speaker
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app: metallb
name: controller
namespace: metallb-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: controller
subjects:
- kind: ServiceAccount
name: controller
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: metallb
component: speaker
name: speaker
namespace: metallb-system
spec:
selector:
matchLabels:
app: metallb
component: speaker
template:
metadata:
annotations:
prometheus.io/port: '7472'
prometheus.io/scrape: 'true'
labels:
app: metallb
component: speaker
spec:
containers:
- args:
- --port=7472
- --config=config
- --log-level=info
env:
- name: METALLB_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: METALLB_HOST
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: METALLB_ML_BIND_ADDR
valueFrom:
fieldRef:
fieldPath: status.podIP
# needed when another software is also using memberlist / port 7946
# when changing this default you also need to update the container ports definition
# and the PodSecurityPolicy hostPorts definition
#- name: METALLB_ML_BIND_PORT
# value: "7946"
- name: METALLB_ML_LABELS
value: "app=metallb,component=speaker"
- name: METALLB_ML_SECRET_KEY
valueFrom:
secretKeyRef:
name: memberlist
key: secretkey
image: quay.io/metallb/speaker:v0.12.1
name: speaker
ports:
- containerPort: 7472
name: monitoring
- containerPort: 7946
name: memberlist-tcp
- containerPort: 7946
name: memberlist-udp
protocol: UDP
livenessProbe:
httpGet:
path: /metrics
port: monitoring
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /metrics
port: monitoring
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
hostNetwork: true
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: speaker
terminationGracePeriodSeconds: 2
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: metallb
component: controller
name: controller
namespace: metallb-system
spec:
revisionHistoryLimit: 3
selector:
matchLabels:
app: metallb
component: controller
template:
metadata:
annotations:
prometheus.io/port: '7472'
prometheus.io/scrape: 'true'
labels:
app: metallb
component: controller
spec:
containers:
- args:
- --port=7472
- --config=config
- --log-level=info
env:
- name: METALLB_ML_SECRET_NAME
value: memberlist
- name: METALLB_DEPLOYMENT
value: controller
image: quay.io/metallb/controller:v0.12.1
name: controller
ports:
- containerPort: 7472
name: monitoring
livenessProbe:
httpGet:
path: /metrics
port: monitoring
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /metrics
port: monitoring
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- all
readOnlyRootFilesystem: true
nodeSelector:
kubernetes.io/os: linux
securityContext:
runAsNonRoot: true
runAsUser: 65534
fsGroup: 65534
serviceAccountName: controller
terminationGracePeriodSeconds: 0

13
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.dual.template.yaml Normal file
View file

@ -0,0 +1,13 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- ${node_ipv4}/32
- ${node_ipv6}/128

12
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-config.ipv4.template.yaml Normal file
View file

@ -0,0 +1,12 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- ${node_ipv4}/32

6
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/metallb-namespace.yaml Normal file
View file

@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: metallb-system
labels:
app: metallb

2
src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/traefik.yaml
View file

@ -19,6 +19,8 @@ spec:
valuesContent: |-
service:
spec:
type: LoadBalancer
externalTrafficPolicy: Local
ipFamilyPolicy: RequireDualStack
rbac:
enabled: true